mod_migrate/mod_migrate.lua
author Kim Alvefur <zash@zash.se>
Tue, 02 May 2023 16:31:25 +0200
changeset 5409 c7a5caad28ef
parent 4402 a16b689525d7
permissions -rw-r--r--
mod_http_oauth2: Enforce response type encoded in client_id The client promises to only use this response type, so we should hold them to that. This makes it fail earlier if the response type is disabled or the client is trying to use one that it promised not to use. Better than failing after login and consent.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
-- mod_migrate
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
     3
local unpack = table.unpack or unpack; --luacheck: ignore 113/unpack
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     4
local sm = require"core.storagemanager";
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
local um = require"core.usermanager";
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
3588
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
     7
local function users(store, host)
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
     8
	if store.users then
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
     9
		return store:users();
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    10
	else
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    11
		return um.users(host);
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    12
	end
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    13
end
611ac62c5b63 mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents: 2820
diff changeset
    14
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    15
local function stores(host)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    16
	if store.users then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    17
		return store:users();
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    18
	else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    19
		return um.users(host);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    20
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    21
end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    22
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    23
local function migrate_store(host, source_store, store_type, migrate_to, migrate_users)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    24
	local module = module:context(host);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    25
	local storage = module:open_store(source_store, store_type);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    26
	local target = assert(sm.load_driver(host, migrate_to));
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    27
	target = assert(target:open(source_store, store_type));
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    28
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    29
	local function migrate_user(username)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    30
		module:log("info", "Migrating %s data for %s", source_store, username);
4402
a16b689525d7 mod_migrate: Allow migrating data not attached to any user / nil user
Kim Alvefur <zash@zash.se>
parents: 4306
diff changeset
    31
		if username == "" then username = nil; end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    32
		local data, err = storage:get(username);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    33
		if not data and err then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    34
			module:log("error", "Could not read data: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    35
		else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    36
			local ok, err = target:set(username, data);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    37
			if not ok then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    38
				module:log("error", "Could not write data: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    39
			end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    40
		end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    41
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    42
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    43
	if store_type == "archive" then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    44
		function migrate_user(username)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    45
			module:log("info", "Migrating %s archive items for %s", source_store, username);
4402
a16b689525d7 mod_migrate: Allow migrating data not attached to any user / nil user
Kim Alvefur <zash@zash.se>
parents: 4306
diff changeset
    46
			if username == "" then username = nil; end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    47
			local count, errs = 0, 0;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    48
			for id, item, when, with in storage:find(username) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    49
				local ok, err = target:append(username, id, item, when, with);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    50
				if ok then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    51
					count = count + 1;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    52
				else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    53
					module:log("warn", "Error: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    54
					errs = errs + 1;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    55
				end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    56
				if ( count + errs ) % 100 == 0 then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    57
					module:log("info", "%d items migrated, %d errors", count, errs);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    58
				end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    59
			end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    60
			module:log("info", "%d items migrated, %d errors", count, errs);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    61
		end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    62
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    63
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    64
	if migrate_users then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    65
		for _, username in ipairs(migrate_users) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    66
			migrate_user(username);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    67
		end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    68
	else
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    69
		xpcall(function()
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    70
			for username in users(storage, host) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    71
				migrate_user(username);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    72
			end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    73
		end,
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    74
		function (err)
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    75
			module:log("error", "Could not list users, you'll have to supply them as arguments");
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    76
			module:log("error", "The error was: %s", err);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    77
		end);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    78
	end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    79
end
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    80
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
function module.command(arg)
2820
cd9831b27746 mod_migrate: Remove unused locals
Kim Alvefur <zash@zash.se>
parents: 2816
diff changeset
    82
	local host, source_stores, migrate_to = unpack(arg);
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    83
	if not migrate_to then
1817
f02f52a2ee11 mod_migrate: Add support for migrating archives
Kim Alvefur <zash@zash.se>
parents: 1816
diff changeset
    84
		return print("Usage: prosodyctl mod_migrate example.com <source-store>[-<store-type>] <target-driver> [users]*");
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    85
	end
2143
17408ddd34b0 mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents: 2068
diff changeset
    86
	if not prosody.hosts[host] then
17408ddd34b0 mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents: 2068
diff changeset
    87
		return print(("The host %q is not know by Prosody."):format(host));
17408ddd34b0 mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents: 2068
diff changeset
    88
	end
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    89
	sm.initialize_host(host);
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    90
	um.initialize_host(host);
1989
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    91
	for source_store in source_stores:gmatch("[^,]+") do
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    92
		local store_type = source_store:match("%-(%a+)$");
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    93
		if store_type then
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    94
			source_store = source_store:sub(1, -2-#store_type);
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
    95
		end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    96
		local migrate_users;
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    97
		if arg[4] then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    98
			migrate_users = {};
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
    99
			for i = 4, #arg do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   100
				migrate_users[i-3] = arg[i];
2068
2a82c55762a7 mod_migrate: Log read and write errors instead of raising error
Kim Alvefur <zash@zash.se>
parents: 2032
diff changeset
   101
			end
1989
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
   102
		end
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   103
		if source_store == "pep_data" then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   104
			for store in sm.get_driver(host, source_store):stores(true) do
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   105
				if store:match("^pep_") then
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   106
					print("Migrating "..store);
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   107
					migrate_store(host, store, store_type, migrate_to, migrate_users);
1817
f02f52a2ee11 mod_migrate: Add support for migrating archives
Kim Alvefur <zash@zash.se>
parents: 1816
diff changeset
   108
				end
1989
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
   109
			end
7821a6986e68 mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents: 1817
diff changeset
   110
		else
4306
c9e1eee6a948 mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents: 3588
diff changeset
   111
			migrate_store(host, source_store, store_type, migrate_to, migrate_users);
1795
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   112
		end
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   113
	end
8df071457dee mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   114
end