mod_export_skeletons/mod_export_skeletons.lua
author Kim Alvefur <zash@zash.se>
Tue, 18 Jan 2022 18:55:20 +0100
changeset 4881 adc6241e5d16
parent 4825 17fbe82d4bfe
permissions -rw-r--r--
mod_measure_process: Report the enforced limit The soft limit is what the kernel actually enforces, while the hard limit is is how far you can change the soft limit without privileges. Unless the process dynamically adjusts the soft limit, knowing the hard limit is not as useful as knowing the soft limit. Reporting the soft limit and the number of in-use FDs allows placing alerts on expressions like 'process_open_fds / process_max_fds >= 0.95'
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4819
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
local t_insert = table.insert;
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     3
local t_sort = table.sort;
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     4
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
local sm = require "core.storagemanager";
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
local um = require "core.usermanager";
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     7
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     8
local argparse = require "util.argparse";
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     9
local dt = require "util.datetime";
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    10
local jid = require "util.jid";
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    11
local st = require "util.stanza";
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    12
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    13
local function skeleton(s)
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    14
	local o = st.stanza(s.name, { xmlns = s.attr.xmlns });
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    15
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    16
	local children = {};
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    17
	for _, child in ipairs(s.tags) do t_insert(children, skeleton(child)) end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    18
	t_sort(children, function(a, b)
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    19
		if a.attr.xmlns == b.attr.xmlns then return a.name < b.name; end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    20
		return (a.attr.xmlns or "") < (b.attr.xmlns or "");
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    21
	end);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    22
	for _, child in ipairs(children) do o:add_direct_child(child); end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    23
	return o;
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    24
end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    25
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    26
local function classify_jid(s)
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    27
	if not s then return "" end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    28
	local u, h, r = jid.split(s);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    29
	if r then
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    30
		return "full"
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    31
	elseif u then
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    32
		return "bare"
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    33
	elseif h then
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    34
		return "host"
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    35
	else
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    36
		return "invalid"
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    37
	end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    38
end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    39
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    40
function module.command(arg)
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    41
	local opts = argparse.parse(arg, { value_params = { store = true; with = true; start = true; ["end"] = true } });
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    42
	local store = opts.store or "archive"; -- so you can pass 'archive2'
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    43
	opts.store = nil;
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    44
	local query = { with = jid.prep(opts.with); start = dt.parse(opts.start); ["end"] = dt.parse(opts["end"]) };
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    45
	local host_initialized = {};
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    46
	for _, export_jid in ipairs(arg) do
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    47
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    48
		local username, host = jid.split(export_jid);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    49
		if not host_initialized[host] then
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    50
			sm.initialize_host(host);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    51
			um.initialize_host(host);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    52
			host_initialized[host] = true;
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    53
		end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    54
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    55
		local archive = module:context(host):open_store(store, "archive");
4825
17fbe82d4bfe mod_export_skeletons: Remove wildcard query
Kim Alvefur <zash@zash.se>
parents: 4824
diff changeset
    56
		local iter, total = assert(archive:find(username, query))
4819
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    57
		if total then io.stderr:write(string.format("Processing %d entries\n", total)); end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    58
		for _, item in iter do
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    59
			local clean = skeleton(item);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    60
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    61
			-- Normalize top level attributes
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    62
			clean.attr.type = item.attr.type;
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    63
			if clean.attr.type == nil and clean.name == "message" then clean.attr.type = "normal"; end
4822
d66162e850cd mod_export_skeletons: Generate ids based on log2 of the original length
Kim Alvefur <zash@zash.se>
parents: 4819
diff changeset
    64
			clean.attr.id = string.rep("x", math.floor(math.log(1+#(item.attr.id or ""), 2)));
4819
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    65
			clean.attr.from = classify_jid(item.attr.from);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    66
			clean.attr.to = classify_jid(item.attr.to);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    67
			print(clean);
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    68
		end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    69
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    70
	end
9c2af2146ee2 mod_export_skeletons: Command to aid in analysis of archive contents
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    71
end