util/statistics.lua
author Matthew Wild <mwild1@gmail.com>
Tue, 20 Jan 2015 12:31:32 +0000
changeset 6556 0e29c05c3503
child 6558 7b2d16c14659
permissions -rw-r--r--
util.statistics: New library for gathering various kinds of statistics
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6556
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     1
local t_sort = table.sort
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     2
local m_floor = math.floor;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     3
local time = require "socket".gettime;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     4
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     5
local function nop_function() end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     6
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     7
local function percentile(arr, length, pc)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     8
	local n = pc/100 * (length + 1);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     9
	local k, d = m_floor(n), n%1;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    10
	if k == 0 then
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    11
		return arr[1];
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    12
	elseif k >= length then
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    13
		return arr[length];
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    14
	end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    15
	return arr[k] + d*(arr[k+1] - arr[k]);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    16
end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    17
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    18
local function new_registry(config)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    19
	config = config or {};
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    20
	local duration_sample_interval = config.duration_sample_interval or 5;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    21
	local duration_max_samples = config.duration_max_stored_samples or 5000;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    22
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    23
	local registry = {};
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    24
	local methods;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    25
	methods = {
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    26
		amount = function (name, initial)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    27
			local v = initial or 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    28
			registry[name] = function () return "amount", v; end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    29
			return function (new_v) v = new_v; end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    30
		end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    31
		counter = function (name, initial)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    32
			local v = initial or 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    33
			registry[name] = function () return "amount", v; end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    34
			return function (delta)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    35
				v = v + delta;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    36
			end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    37
		end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    38
		rate = function (name)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    39
			local since, n = time(), 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    40
			registry[name] = function ()
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    41
				local t = time();
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    42
				local stats = {
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    43
					rate = n/(t-since);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    44
					count = n;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    45
				};
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    46
				since, n = t, 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    47
				return "rate", stats.rate, stats;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    48
			end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    49
			return function ()
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    50
				n = n + 1;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    51
			end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    52
		end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    53
		duration = function (name)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    54
			local events, last_event = {}, 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    55
			local n_actual_events = 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    56
			local since = time();
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    57
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    58
			registry[name] = function ()
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    59
				local n_stored_events = #events;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    60
				t_sort(events);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    61
				local sum = 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    62
				for i = 1, n_stored_events do
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    63
					sum = sum + events[i];
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    64
				end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    65
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    66
				local new_time = time();
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    67
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    68
				local stats = {
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    69
					samples = events;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    70
					sample_count = n_stored_events;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    71
					count = n_actual_events,
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    72
					rate = n_actual_events/(new_time-since);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    73
					average = n_stored_events > 0 and sum/n_stored_events or 0,
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    74
					min = events[1],
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    75
					max = events[n_stored_events],
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    76
				};
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    77
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    78
				events, last_event = {}, 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    79
				n_actual_events = 0;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    80
				since = new_time;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    81
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    82
				return "duration", stats.average, stats;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    83
			end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    84
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    85
			return function ()
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    86
				n_actual_events = n_actual_events + 1;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    87
				if n_actual_events%duration_sample_interval > 0 then
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    88
					return nop_function;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    89
				end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    90
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    91
				local start_time = time();
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    92
				return function ()
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    93
					local end_time = time();
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    94
					local duration = end_time - start_time;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    95
					last_event = (last_event%duration_max_samples) + 1;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    96
					events[last_event] = duration;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    97
				end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    98
			end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    99
		end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   100
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   101
		get_stats = function ()
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   102
			return registry;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   103
		end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   104
	};
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   105
	return methods;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   106
end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   107
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   108
return {
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   109
	new = new_registry;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   110
	get_histogram = function (duration, n_buckets)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   111
		n_buckets = n_buckets or 100;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   112
		local events, n_events = duration.samples, duration.sample_count;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   113
		if not (events and n_events) then
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   114
			return nil, "not a valid duration stat";
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   115
		end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   116
		local histogram = {};
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   117
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   118
		for i = 1, 100, 100/n_buckets do
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   119
			histogram[i] = percentile(events, n_events, i);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   120
		end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   121
		return histogram;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   122
	end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   123
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   124
	get_percentile = function (duration, pc)
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   125
		local events, n_events = duration.samples, duration.sample_count;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   126
		if not (events and n_events) then
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   127
			return nil, "not a valid duration stat";
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   128
		end
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   129
		return percentile(events, n_events, pc);
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   130
	end;
0e29c05c3503 util.statistics: New library for gathering various kinds of statistics
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
   131
}