mod_storage_lmdb/mod_storage_lmdb.lua
author Kim Alvefur <zash@zash.se>
Fri, 28 Aug 2015 00:38:19 +0200
changeset 1804 669d1208221a
parent 1803 d2dd1db9ece6
child 1985 1f815f57fa57
permissions -rw-r--r--
mod_storage_lmdb: Remove redundant table
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
-- mod_storage_lmdb
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
-- Copyright (C) 2015 Kim Alvefur
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     3
--
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     4
-- This file is MIT/X11 licensed.
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
-- 
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
-- Depends on lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     7
-- https://github.com/shmul/lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     8
--
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     9
-- luacheck: globals prosody open
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    10
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    11
local lmdb = require"lightningmdb";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    12
local lfs = require"lfs";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    13
local path = require"util.paths";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    14
local serialization = require"util.serialization";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    15
local serialize = serialization.serialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    16
local deserialize = serialization.deserialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    17
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    18
local drivers = {};
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    19
local provider = {};
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    20
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    21
local keyval = {};
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    22
local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    23
drivers.keyval = keyval_mt;
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    24
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    25
function keyval:set(user, value)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    26
	local t = self.env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    27
	if type(value) == "table" and next(value) == nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    28
		value = nil;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    29
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    30
	if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    31
		value = serialize(value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    32
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    33
	local ok, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    34
	if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    35
		ok, err = t:put(self.db, user, value, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    36
	else
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    37
		ok, err = t:del(self.db, user, value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    38
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    39
	if not ok then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    40
		t:abort();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    41
		return nil, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    42
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    43
	return t:commit();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    44
end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    45
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    46
function keyval:get(user)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    47
	local t = self.env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    48
	local data, err = t:get(self.db, user, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    49
	if not data then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    50
		t:abort();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    51
		return nil, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    52
	end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    53
	t:commit();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    54
	return deserialize(data);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    55
end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    56
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    57
function provider:init(config)
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    58
	if config.base_path then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    59
		lfs.mkdir(config.base_path);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    60
	end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    61
	local env = lmdb.env_create();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    62
	env:set_maxdbs(config.maxdbs or 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    63
	local env_flags = 0;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    64
	if config.flags then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    65
		for flag in config.flags do
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    66
			env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    67
		end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    68
	end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    69
	env:open(config.base_path or ".", env_flags, tonumber("640", 8));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    70
	self.env = env;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    71
end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    72
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    73
function provider:open(store, typ)
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    74
	typ = typ or "keyval";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    75
	local driver_mt = drivers[typ];
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    76
	if not driver_mt then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    77
		return nil, "unsupported-store";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    78
	end
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    79
	local env = self.env;
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    80
	local t = env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
	local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    82
	local ok, err = t:commit();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    83
	if not ok then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    84
		module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    85
		return ok, err;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    86
	end
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    87
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    88
	return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    89
end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    90
1803
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    91
if prosody then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    92
	provider:init({
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    93
		base_path = path.resolve_relative_path(prosody.paths.data, module.host);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    94
		flags = module:get_option_set("lmdb_flags", {});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    95
		maxdbs = module:get_option_number("lmdb_maxdbs", 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    96
	});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    97
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    98
	function module.unload()
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
    99
		provider.env:sync(1);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   100
		provider.env:close();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   101
	end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   102
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   103
	module:provides("storage", provider);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   104
else
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
   105
	return provider;
1759
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   106
end