author | Kim Alvefur <zash@zash.se> |
Fri, 04 Sep 2015 00:23:46 +0200 | |
changeset 1831 | 9376e870f0e1 |
parent 1804 | 669d1208221a |
child 1985 | 1f815f57fa57 |
permissions | -rw-r--r-- |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 |
-- mod_storage_lmdb |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 |
-- Copyright (C) 2015 Kim Alvefur |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 |
-- |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 |
-- This file is MIT/X11 licensed. |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 |
-- |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 |
-- Depends on lightningdbm |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 |
-- https://github.com/shmul/lightningdbm |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 |
-- |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 |
-- luacheck: globals prosody open |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 |
|
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
11 |
local lmdb = require"lightningmdb"; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
12 |
local lfs = require"lfs"; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
13 |
local path = require"util.paths"; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
14 |
local serialization = require"util.serialization"; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 |
local serialize = serialization.serialize; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
16 |
local deserialize = serialization.deserialize; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
17 |
|
1803
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
18 |
local drivers = {}; |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
19 |
local provider = {}; |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
20 |
|
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
21 |
local keyval = {}; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
22 |
local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE }; |
1803
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
23 |
drivers.keyval = keyval_mt; |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
24 |
|
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
25 |
function keyval:set(user, value) |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
26 |
local t = self.env:txn_begin(nil, 0); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 |
if type(value) == "table" and next(value) == nil then |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 |
value = nil; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 |
if value ~= nil then |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
31 |
value = serialize(value); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
32 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
33 |
local ok, err; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
34 |
if value ~= nil then |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
35 |
ok, err = t:put(self.db, user, value, 0); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
36 |
else |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 |
ok, err = t:del(self.db, user, value); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
38 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
39 |
if not ok then |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
40 |
t:abort(); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
41 |
return nil, err; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
42 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
43 |
return t:commit(); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
44 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
45 |
|
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
46 |
function keyval:get(user) |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
47 |
local t = self.env:txn_begin(nil, 0); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
48 |
local data, err = t:get(self.db, user, 0); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
49 |
if not data then |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
50 |
t:abort(); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
51 |
return nil, err; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
52 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
53 |
t:commit(); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
54 |
return deserialize(data); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
55 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
56 |
|
1803
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
57 |
function provider:init(config) |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
58 |
if config.base_path then |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
59 |
lfs.mkdir(config.base_path); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
60 |
end |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
61 |
local env = lmdb.env_create(); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
62 |
env:set_maxdbs(config.maxdbs or 20); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
63 |
local env_flags = 0; |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
64 |
if config.flags then |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
65 |
for flag in config.flags do |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
66 |
env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
67 |
end |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
68 |
end |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
69 |
env:open(config.base_path or ".", env_flags, tonumber("640", 8)); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
70 |
self.env = env; |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
71 |
end |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
72 |
|
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
73 |
function provider:open(store, typ) |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
74 |
typ = typ or "keyval"; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
75 |
local driver_mt = drivers[typ]; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
76 |
if not driver_mt then |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
77 |
return nil, "unsupported-store"; |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
78 |
end |
1803
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
79 |
local env = self.env; |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
80 |
local t = env:txn_begin(nil, 0); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
81 |
local db = t:dbi_open(store.."_"..typ, driver_mt.flags); |
1803
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
82 |
local ok, err = t:commit(); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
83 |
if not ok then |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
84 |
module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err)); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
85 |
return ok, err; |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
86 |
end |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
87 |
|
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
88 |
return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt); |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
89 |
end |
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
90 |
|
1803
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
91 |
if prosody then |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
92 |
provider:init({ |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
93 |
base_path = path.resolve_relative_path(prosody.paths.data, module.host); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
94 |
flags = module:get_option_set("lmdb_flags", {}); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
95 |
maxdbs = module:get_option_number("lmdb_maxdbs", 20); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
96 |
}); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
97 |
|
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
98 |
function module.unload() |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
99 |
provider.env:sync(1); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
100 |
provider.env:close(); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
101 |
end |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
102 |
|
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
103 |
module:provides("storage", provider); |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
104 |
else |
d2dd1db9ece6
mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents:
1759
diff
changeset
|
105 |
return provider; |
1759
0a21b16b9075
mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
106 |
end |