author | Matthew Wild <mwild1@gmail.com> |
Thu, 28 Jan 2021 07:04:11 +0000 | |
changeset 4413 | 44f6537f6427 |
parent 4402 | a16b689525d7 |
permissions | -rw-r--r-- |
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 |
-- mod_migrate |
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 |
|
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
3 |
local unpack = table.unpack or unpack; --luacheck: ignore 113/unpack |
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 |
local sm = require"core.storagemanager"; |
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 |
local um = require"core.usermanager"; |
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 |
|
3588
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
7 |
local function users(store, host) |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
8 |
if store.users then |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
9 |
return store:users(); |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
10 |
else |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
11 |
return um.users(host); |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
12 |
end |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
13 |
end |
611ac62c5b63
mod_migrate: Try to use per storage user listing where available
Kim Alvefur <zash@zash.se>
parents:
2820
diff
changeset
|
14 |
|
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
15 |
local function stores(host) |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
16 |
if store.users then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
17 |
return store:users(); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
18 |
else |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
19 |
return um.users(host); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
20 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
21 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
22 |
|
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
23 |
local function migrate_store(host, source_store, store_type, migrate_to, migrate_users) |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
24 |
local module = module:context(host); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
25 |
local storage = module:open_store(source_store, store_type); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
26 |
local target = assert(sm.load_driver(host, migrate_to)); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
27 |
target = assert(target:open(source_store, store_type)); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
28 |
|
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
29 |
local function migrate_user(username) |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
30 |
module:log("info", "Migrating %s data for %s", source_store, username); |
4402
a16b689525d7
mod_migrate: Allow migrating data not attached to any user / nil user
Kim Alvefur <zash@zash.se>
parents:
4306
diff
changeset
|
31 |
if username == "" then username = nil; end |
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
32 |
local data, err = storage:get(username); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
33 |
if not data and err then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
34 |
module:log("error", "Could not read data: %s", err); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
35 |
else |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
36 |
local ok, err = target:set(username, data); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
37 |
if not ok then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
38 |
module:log("error", "Could not write data: %s", err); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
39 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
40 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
41 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
42 |
|
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
43 |
if store_type == "archive" then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
44 |
function migrate_user(username) |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
45 |
module:log("info", "Migrating %s archive items for %s", source_store, username); |
4402
a16b689525d7
mod_migrate: Allow migrating data not attached to any user / nil user
Kim Alvefur <zash@zash.se>
parents:
4306
diff
changeset
|
46 |
if username == "" then username = nil; end |
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
47 |
local count, errs = 0, 0; |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
48 |
for id, item, when, with in storage:find(username) do |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
49 |
local ok, err = target:append(username, id, item, when, with); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
50 |
if ok then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
51 |
count = count + 1; |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
52 |
else |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
53 |
module:log("warn", "Error: %s", err); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
54 |
errs = errs + 1; |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
55 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
56 |
if ( count + errs ) % 100 == 0 then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
57 |
module:log("info", "%d items migrated, %d errors", count, errs); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
58 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
59 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
60 |
module:log("info", "%d items migrated, %d errors", count, errs); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
61 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
62 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
63 |
|
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
64 |
if migrate_users then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
65 |
for _, username in ipairs(migrate_users) do |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
66 |
migrate_user(username); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
67 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
68 |
else |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
69 |
xpcall(function() |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
70 |
for username in users(storage, host) do |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
71 |
migrate_user(username); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
72 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
73 |
end, |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
74 |
function (err) |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
75 |
module:log("error", "Could not list users, you'll have to supply them as arguments"); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
76 |
module:log("error", "The error was: %s", err); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
77 |
end); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
78 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
79 |
end |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
80 |
|
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
81 |
function module.command(arg) |
2820
cd9831b27746
mod_migrate: Remove unused locals
Kim Alvefur <zash@zash.se>
parents:
2816
diff
changeset
|
82 |
local host, source_stores, migrate_to = unpack(arg); |
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
83 |
if not migrate_to then |
1817
f02f52a2ee11
mod_migrate: Add support for migrating archives
Kim Alvefur <zash@zash.se>
parents:
1816
diff
changeset
|
84 |
return print("Usage: prosodyctl mod_migrate example.com <source-store>[-<store-type>] <target-driver> [users]*"); |
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
85 |
end |
2143
17408ddd34b0
mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents:
2068
diff
changeset
|
86 |
if not prosody.hosts[host] then |
17408ddd34b0
mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents:
2068
diff
changeset
|
87 |
return print(("The host %q is not know by Prosody."):format(host)); |
17408ddd34b0
mod_migrate: Check that the hostname is known, or yell and abort (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents:
2068
diff
changeset
|
88 |
end |
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
89 |
sm.initialize_host(host); |
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
90 |
um.initialize_host(host); |
1989
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
91 |
for source_store in source_stores:gmatch("[^,]+") do |
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
92 |
local store_type = source_store:match("%-(%a+)$"); |
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
93 |
if store_type then |
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
94 |
source_store = source_store:sub(1, -2-#store_type); |
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
95 |
end |
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
96 |
local migrate_users; |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
97 |
if arg[4] then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
98 |
migrate_users = {}; |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
99 |
for i = 4, #arg do |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
100 |
migrate_users[i-3] = arg[i]; |
2068
2a82c55762a7
mod_migrate: Log read and write errors instead of raising error
Kim Alvefur <zash@zash.se>
parents:
2032
diff
changeset
|
101 |
end |
1989
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
102 |
end |
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
103 |
if source_store == "pep_data" then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
104 |
for store in sm.get_driver(host, source_store):stores(true) do |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
105 |
if store:match("^pep_") then |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
106 |
print("Migrating "..store); |
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
107 |
migrate_store(host, store, store_type, migrate_to, migrate_users); |
1817
f02f52a2ee11
mod_migrate: Add support for migrating archives
Kim Alvefur <zash@zash.se>
parents:
1816
diff
changeset
|
108 |
end |
1989
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
109 |
end |
7821a6986e68
mod_migrate: Support migrating multiple stores
Kim Alvefur <zash@zash.se>
parents:
1817
diff
changeset
|
110 |
else |
4306
c9e1eee6a948
mod_migrate: Add support for the fictitious pep_data store
Matthew Wild <mwild1@gmail.com>
parents:
3588
diff
changeset
|
111 |
migrate_store(host, source_store, store_type, migrate_to, migrate_users); |
1795
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
112 |
end |
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
113 |
end |
8df071457dee
mod_migrate: Provides a prosodyctl mod_migrate command for copying data between storage backends
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
114 |
end |