teal-src/plugins/mod_cron.tl
author Kim Alvefur <zash@zash.se>
Sun, 21 Nov 2021 15:50:36 +0100
changeset 11990 3d5135e8a2a7
child 11999 bbd3ac65640d
permissions -rw-r--r--
mod_cron: Initial commit of periodic task runner A number of modules now have periodic tasks that need to run, e.g. for cleaning out old messages or files. This has highlighted a need for coordinating and optimizing scheduling of such tasks.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
11990
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
module:set_global();
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     3
local async = require "util.async";
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     4
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
local record map_store<K,V>
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
	-- TODO move to somewhere sensible
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     7
	get : function (map_store<K,V>, string, K) : V
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     8
	set : function (map_store<K,V>, string, K, V)
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     9
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    10
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    11
local enum frequency
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    12
	"hourly"
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    13
	"daily"
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    14
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    15
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    16
local record task_spec
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    17
	id : string -- unique id
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    18
	name : string -- name or short description
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    19
	when : frequency
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    20
	last : integer
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    21
	run : function (task_spec, integer)
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    22
	save : function (task_spec, integer)
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    23
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    24
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    25
local record task_event
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    26
	source : module
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    27
	item : task_spec
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    28
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    29
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    30
local periods : { frequency : integer } = { hourly = 3600, daily = 86400 }
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    31
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    32
local active_hosts : { string : boolean } = {  }
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    33
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    34
function module.add_host(host_module : moduleapi)
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    35
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    36
	local last_run_times = host_module:open_store("cron", "map") as map_store<string,integer>;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    37
	active_hosts[host_module.host] = true;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    38
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    39
	local function save_task(task : task_spec, started_at : integer)
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    40
		last_run_times:set(nil, task.id, started_at);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    41
	end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    42
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    43
	local function task_added(event : task_event) : boolean
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    44
		local task = event.item;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    45
		if task.name == nil then
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    46
			task.name = task.when;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    47
		end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    48
		if task.id == nil then
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    49
			task.id = event.source.name .. "/" .. task.name:gsub("%W", "_"):lower();
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    50
		end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    51
		if task.last == nil then
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    52
			task.last = last_run_times:get(nil, task.id);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    53
		end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    54
		task.save = save_task;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    55
		module:log("debug", "%s task %s added, last run %s", task.when, task.id,
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    56
			task.last and require"util.datetime".datetime(task.last) or "never");
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    57
		return true;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    58
	end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    59
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    60
	local function task_removed(event : task_event) : boolean
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    61
		local task = event.item;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    62
		host_module:log("debug", "Task %s removed", task.id);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    63
		return true;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    64
	end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    65
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    66
	host_module:handle_items("task", task_added, task_removed, true);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    67
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    68
	function host_module.unload()
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    69
		active_hosts[host_module.host]=nil;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    70
	end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    71
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    72
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    73
local function should_run(when : frequency, last : integer) : boolean
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    74
	return not last or last + periods[when] <= os.time();
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    75
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    76
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    77
local function run_task(task : task_spec)
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    78
	local started_at = os.time();
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    79
	task:run(started_at);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    80
	task:save(started_at);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    82
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    83
local task_runner = async.runner(run_task);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    84
module:add_timer(1, function() : integer
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    85
	module:log("info", "Running periodic tasks");
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    86
	local delay = 3600;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    87
	for host in pairs(active_hosts) do
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    88
		module:log("debug", "Running periodic tasks for host %s", host);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    89
		for _, task in ipairs(module:context(host):get_host_items("task") as { task_spec } ) do
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    90
			module:log("debug", "Considering %s task %s (%s)", task.when, task.id, task.run);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    91
			if should_run(task.when, task.last) then task_runner:run(task); end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    92
		end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    93
	end
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    94
	module:log("debug", "Wait %ds", delay);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    95
	return delay;
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    96
end);
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    97
3d5135e8a2a7 mod_cron: Initial commit of periodic task runner
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    98
-- TODO measure load, pick a good time to do stuff