mod_log_ringbuffer/mod_log_ringbuffer.lua
author Matthew Wild <mwild1@gmail.com>
Tue, 20 Oct 2020 15:34:29 +0100
changeset 4229 d6fb9f7afaa5
parent 4223 b3dd77f2d0d7
child 4230 df2ccb42a241
permissions -rw-r--r--
mod_log_ringbuffer: Discard old data when buffer is full
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4209
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     1
module:set_global();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     2
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     3
local loggingmanager = require "core.loggingmanager";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     4
local format = require "util.format".format;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     5
local pposix = require "util.pposix";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     6
local rb = require "util.ringbuffer";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     7
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     8
local default_timestamp = "%b %d %H:%M:%S ";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     9
local max_chunk_size = module:get_option_number("log_ringbuffer_chunk_size", 16384);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    10
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    11
local os_date = os.date;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    12
4219
86f8ece24029 mod_log_ringbuffer: Update default filename to include data path
Matthew Wild <mwild1@gmail.com>
parents: 4209
diff changeset
    13
local default_filename_template = "{paths.data}/ringbuffer-logs-{pid}-{count}.log";
4209
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    14
local render_filename = require "util.interpolation".new("%b{}", function (s) return s; end, {
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    15
	yyyymmdd = function (t)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    16
		return os_date("%Y%m%d", t);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    17
	end;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    18
	hhmmss = function (t)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    19
		return os_date("%H%M%S", t);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    20
	end;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    21
});
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    22
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    23
local dump_count = 0;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    24
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    25
local function dump_buffer(buffer, filename)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    26
	dump_count = dump_count + 1;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    27
	local f, err = io.open(filename, "a+");
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    28
	if not f then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    29
		module:log("error", "Unable to open output file: %s", err);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    30
		return;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    31
	end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    32
	local bytes_remaining = buffer:length();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    33
	f:write(("-- Dumping %d bytes at %s --\n"):format(bytes_remaining, os_date(default_timestamp)));
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    34
	while bytes_remaining > 0 do
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    35
		local chunk_size = math.min(bytes_remaining, max_chunk_size);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    36
		local chunk = buffer:read(chunk_size);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    37
		if not chunk then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    38
			f:write("-- Dump aborted due to error --\n\n");
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    39
			f:close();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    40
			return;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    41
		end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    42
		f:write(chunk);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    43
		bytes_remaining = bytes_remaining - chunk_size;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    44
	end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    45
	f:write("-- End of dump --\n\n");
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    46
	f:close();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    47
end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    48
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    49
local function get_filename(filename_template)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    50
	filename_template = filename_template or default_filename_template;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    51
	return render_filename(filename_template, {
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    52
		paths = prosody.paths;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    53
		pid = pposix.getpid();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    54
		count = dump_count;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    55
		time = os.time();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    56
	});
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    57
end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    58
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    59
local function ringbuffer_log_sink_maker(sink_config)
4229
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    60
	local buffer_size = sink_config.size or 100*1024;
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    61
	local buffer = rb.new(buffer_size);
4209
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    62
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    63
	local timestamps = sink_config.timestamps;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    64
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    65
	if timestamps == true or timestamps == nil then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    66
		timestamps = default_timestamp; -- Default format
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    67
	elseif timestamps then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    68
		timestamps = timestamps .. " ";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    69
	end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    70
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    71
	local function dump()
4223
b3dd77f2d0d7 mod_log_ringbuffer: Switch `filename` to not be interpolated, add filename_template which is
Matthew Wild <mwild1@gmail.com>
parents: 4222
diff changeset
    72
		dump_buffer(buffer, sink_config.filename or get_filename(sink_config.filename_template));
4209
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    73
	end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    74
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    75
	if sink_config.signal then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    76
		require "util.signal".signal(sink_config.signal, dump);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    77
	elseif sink_config.event then
4222
f917bb78ab67 mod_log_ringbuffer: Fix incorrect parameter name
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
    78
		module:hook_global(sink_config.event, dump);
4209
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    79
	end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    80
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    81
	return function (name, level, message, ...)
4229
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    82
		local line = format("%s%s\t%s\t%s\n", timestamps and os_date(timestamps) or "", name, level, format(message, ...));
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    83
		if not buffer:write(line) then
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    84
			if #line > buffer_size then
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    85
				buffer:discard(buffer_size);
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    86
				buffer:write(line:sub(-buffer_size));
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    87
			else
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    88
				buffer:discard(#line);
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    89
				buffer:write(line);
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    90
			end
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4223
diff changeset
    91
		end
4209
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    92
	end;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    93
end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    94
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    95
loggingmanager.register_sink_type("ringbuffer", ringbuffer_log_sink_maker);