mod_migrate/mod_migrate.lua
author Matthew Wild <mwild1@gmail.com>
Sat, 24 Sep 2022 09:26:26 +0100
changeset 5063 5f1120c284c5
parent 4402 a16b689525d7
permissions -rw-r--r--
mod_cloud_notify_extensions: Add note about dependency Noting here because people might not click through to see it on the mod_cloud_notify_encrypted page.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
-- mod_migrate
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
     3
local unpack = table.unpack or unpack; --luacheck: ignore 113/unpack
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     4
local sm = require"core.storagemanager";
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
local um = require"core.usermanager";
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
3588
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
     7
local function users(store, host)
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
     8
	if store.users then
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
     9
		return store:users();
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    10
	else
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    11
		return um.users(host);
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    12
	end
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    13
end
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    14
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    15
local function stores(host)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    16
	if store.users then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    17
		return store:users();
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    18
	else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    19
		return um.users(host);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    20
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    21
end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    22
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    23
local function migrate_store(host, source_store, store_type, migrate_to, migrate_users)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    24
	local module = module:context(host);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    25
	local storage = module:open_store(source_store, store_type);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    26
	local target = assert(sm.load_driver(host, migrate_to));
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    27
	target = assert(target:open(source_store, store_type));
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    28
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    29
	local function migrate_user(username)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    30
		module:log("info", "Migrating %s data for %s", source_store, username);
4402
a16b689525d7 mod_migrate: Allow migrating data not attached to any user / nil user
Kim Alvefur <zash@zash.se>
parents: 4306
diff changeset
    31
		if username == "" then username = nil; end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    32
		local data, err = storage:get(username);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    33
		if not data and err then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    34
			module:log("error", "Could not read data: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    35
		else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    36
			local ok, err = target:set(username, data);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    37
			if not ok then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    38
				module:log("error", "Could not write data: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    39
			end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    40
		end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    41
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    42
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    43
	if store_type == "archive" then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    44
		function migrate_user(username)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    45
			module:log("info", "Migrating %s archive items for %s", source_store, username);
4402
a16b689525d7 mod_migrate: Allow migrating data not attached to any user / nil user
Kim Alvefur <zash@zash.se>
parents: 4306
diff changeset
    46
			if username == "" then username = nil; end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    47
			local count, errs = 0, 0;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    48
			for id, item, when, with in storage:find(username) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    49
				local ok, err = target:append(username, id, item, when, with);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    50
				if ok then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    51
					count = count + 1;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    52
				else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    53
					module:log("warn", "Error: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    54
					errs = errs + 1;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    55
				end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    56
				if ( count + errs ) % 100 == 0 then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    57
					module:log("info", "%d items migrated, %d errors", count, errs);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    58
				end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    59
			end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    60
			module:log("info", "%d items migrated, %d errors", count, errs);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    61
		end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    62
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    63
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    64
	if migrate_users then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    65
		for _, username in ipairs(migrate_users) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    66
			migrate_user(username);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    67
		end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    68
	else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    69
		xpcall(function()
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    70
			for username in users(storage, host) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    71
				migrate_user(username);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    72
			end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    73
		end,
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    74
		function (err)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    75
			module:log("error", "Could not list users, you'll have to supply them as arguments");
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    76
			module:log("error", "The error was: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    77
		end);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    78
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    79
end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    80
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
function module.command(arg)
2820
cd9831b27746 mod_migrate: Remove unused locals
Kim Alvefur <zash@zash.se>
parents: 2816
diff changeset
    82
	local host, source_stores, migrate_to = unpack(arg);
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    83
	if not migrate_to then
1817
f02f52a2ee11 mod_migrate: Add support for migrating archives
Kim Alvefur <zash@zash.se>
parents: 1816
diff changeset
    84
		return print("Usage: prosodyctl mod_migrate example.com <source-store>[-<store-type>] <target-driver> [users]*");
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    85
	end
2143
17408ddd34b0 mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents: 2068
diff changeset
    86
	if not prosody.hosts[host] then
17408ddd34b0 mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents: 2068
diff changeset
    87
		return print(("The host %q is not know by Prosody."):format(host));
17408ddd34b0 mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents: 2068
diff changeset
    88
	end
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    89
	sm.initialize_host(host);
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    90
	um.initialize_host(host);
1989
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    91
	for source_store in source_stores:gmatch("[^,]+") do
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    92
		local store_type = source_store:match("%-(%a+)$");
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    93
		if store_type then
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    94
			source_store = source_store:sub(1, -2-#store_type);
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    95
		end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    96
		local migrate_users;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    97
		if arg[4] then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    98
			migrate_users = {};
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    99
			for i = 4, #arg do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   100
				migrate_users[i-3] = arg[i];
2068
2a82c55762a7 mod_migrate: Log read and write errors instead of raising error
Kim Alvefur <zash@zash.se>
parents: 2032
diff changeset
   101
			end
1989
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
   102
		end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   103
		if source_store == "pep_data" then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   104
			for store in sm.get_driver(host, source_store):stores(true) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   105
				if store:match("^pep_") then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   106
					print("Migrating "..store);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   107
					migrate_store(host, store, store_type, migrate_to, migrate_users);
1817
f02f52a2ee11 mod_migrate: Add support for migrating archives
Kim Alvefur <zash@zash.se>
parents: 1816
diff changeset
   108
				end
1989
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
   109
			end
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
   110
		else
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   111
			migrate_store(host, source_store, store_type, migrate_to, migrate_users);
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   112
		end
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   113
	end
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   114
end