mod_storage_lmdb/mod_storage_lmdb.lua
author Kim Alvefur <zash@zash.se>
Fri, 28 Aug 2015 00:34:43 +0200
changeset 1803 d2dd1db9ece6
parent 1759 0a21b16b9075
child 1804 669d1208221a
permissions -rw-r--r--
mod_storage_lmdb: Rearrange module to allow using as a library
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
-- mod_storage_lmdb
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
-- Copyright (C) 2015 Kim Alvefur
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     3
--
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     4
-- This file is MIT/X11 licensed.
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
-- 
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
-- Depends on lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     7
-- https://github.com/shmul/lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     8
--
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     9
-- luacheck: globals prosody open
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    10
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    11
local lmdb = require"lightningmdb";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    12
local lfs = require"lfs";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    13
local path = require"util.paths";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    14
local serialization = require"util.serialization";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    15
local serialize = serialization.serialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    16
local deserialize = serialization.deserialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    17
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    18
local drivers = {};
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    19
local provider = {};
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    20
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    21
local keyval = {};
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    22
local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    23
drivers.keyval = keyval_mt;
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    24
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    25
function keyval:set(user, value)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    26
	local t = self.env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    27
	if type(value) == "table" and next(value) == nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    28
		value = nil;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    29
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    30
	if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    31
		value = serialize(value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    32
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    33
	local ok, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    34
	if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    35
		ok, err = t:put(self.db, user, value, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    36
	else
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    37
		ok, err = t:del(self.db, user, value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    38
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    39
	if not ok then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    40
		t:abort();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    41
		return nil, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    42
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    43
	return t:commit();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    44
end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    45
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    46
function keyval:get(user)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    47
	local t = self.env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    48
	local data, err = t:get(self.db, user, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    49
	if not data then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    50
		t:abort();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    51
		return nil, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    52
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    53
	t:commit();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    54
	return deserialize(data);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    55
end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    56
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    57
local drivers = {
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    58
	keyval = keyval_mt;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    59
}
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    60
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    61
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    62
function provider:init(config)
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    63
	if config.base_path then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    64
		lfs.mkdir(config.base_path);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    65
	end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    66
	local env = lmdb.env_create();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    67
	env:set_maxdbs(config.maxdbs or 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    68
	local env_flags = 0;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    69
	if config.flags then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    70
		for flag in config.flags do
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    71
			env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    72
		end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    73
	end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    74
	env:open(config.base_path or ".", env_flags, tonumber("640", 8));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    75
	self.env = env;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    76
end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    77
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    78
function provider:open(store, typ)
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    79
	typ = typ or "keyval";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    80
	local driver_mt = drivers[typ];
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
	if not driver_mt then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    82
		return nil, "unsupported-store";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    83
	end
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    84
	local env = self.env;
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    85
	local t = env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    86
	local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    87
	local ok, err = t:commit();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    88
	if not ok then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    89
		module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    90
		return ok, err;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    91
	end
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    92
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    93
	return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    94
end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    95
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    96
if prosody then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    97
	provider:init({
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    98
		base_path = path.resolve_relative_path(prosody.paths.data, module.host);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    99
		flags = module:get_option_set("lmdb_flags", {});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   100
		maxdbs = module:get_option_number("lmdb_maxdbs", 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   101
	});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   102
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   103
	function module.unload()
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   104
		provider.env:sync(1);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   105
		provider.env:close();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   106
	end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   107
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   108
	module:provides("storage", provider);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   109
else
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   110
	return provider;
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   111
end