mod_http_file_share: Always measure total disk usage for statistics!
authorKim Alvefur <zash@zash.se>
Tue, 11 Jan 2022 04:15:29 +0100
changeset 12183 5e68635cdc2c
parent 12182 0aa99a6dfb3e
child 12184 53e0ae770917
mod_http_file_share: Always measure total disk usage for statistics! Metrics available or not depending on configuration is weird, even tho it might be expensive to calculate and it's only really needed when there is a global quota. Default quota is set to infinity, which is essentially what it was. Reports NaN if there is an error, which should count as over the infinite default quota.
plugins/mod_http_file_share.lua
--- a/plugins/mod_http_file_share.lua	Tue Jan 11 00:06:48 2022 +0100
+++ b/plugins/mod_http_file_share.lua	Tue Jan 11 04:15:29 2022 +0100
@@ -20,6 +20,9 @@
 local cache = require "util.cache";
 local lfs = require "lfs";
 
+local unknown = math.abs(0/0);
+local unlimited = math.huge;
+
 local namespace = "urn:xmpp:http:upload:0";
 
 module:depends("disco");
@@ -38,7 +41,7 @@
 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
-local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
+local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited);
 
 local access = module:get_option_set(module.name .. "_access", {});
 
@@ -60,30 +63,29 @@
 	};
 	filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
 	quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
-	unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
 	outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
 });
 
 local upload_cache = cache.new(1024);
 local quota_cache = cache.new(1024);
 
-local total_storage_usage = nil;
+local total_storage_usage = unknown;
 
 local measure_upload_cache_size = module:measure("upload_cache", "amount");
 local measure_quota_cache_size = module:measure("quota_cache", "amount");
-local measure_total_storage_usage = nil;
-if total_storage_limit then
+local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
+
+do
 	local total, err = persist_stats:get(nil, "total");
-	if not err then total_storage_usage = tonumber(total) or 0; end
-	measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
+	if not err then
+		total_storage_usage = tonumber(total) or 0;
+	end
 end
 
 module:hook_global("stats-update", function ()
 	measure_upload_cache_size(upload_cache:count());
 	measure_quota_cache_size(quota_cache:count());
-	if total_storage_limit and total_storage_usage then
-		measure_total_storage_usage(total_storage_usage);
-	end
+	measure_total_storage_usage(total_storage_usage);
 end);
 
 local buckets = {};
@@ -95,7 +97,14 @@
 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets});
 
 -- Convenience wrapper for logging file sizes
-local function B(bytes) return hi.format(bytes, "B", "b"); end
+local function B(bytes)
+	if bytes ~= bytes then
+		return "unknown"
+	elseif bytes == unlimited then
+		return "unlimited";
+	end
+	return hi.format(bytes, "B", "b");
+end
 
 local function get_filename(slot, create)
 	return dm.getpath(slot, module.host, module.name, "bin", create)
@@ -141,13 +150,9 @@
 		return false, upload_errors.new("filesize");
 	end
 
-	if total_storage_limit then
-		if not total_storage_usage  then
-			return false, upload_errors.new("unknowntotal");
-		elseif total_storage_usage + filesize > total_storage_limit then
-			module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
-			return false, upload_errors.new("outofdisk");
-		end
+	if total_storage_usage + filesize > total_storage_limit then
+		module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit));
+		return false, upload_errors.new("outofdisk");
 	end
 
 	local uploader_quota = get_daily_quota(uploader);
@@ -217,10 +222,8 @@
 		return true;
 	end
 
-	if total_storage_usage then
-		total_storage_usage = total_storage_usage + filesize;
-		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
-	end
+	total_storage_usage = total_storage_usage + filesize;
+	module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
 
 	local cached_quota = quota_cache:get(uploader);
 	if cached_quota and cached_quota.time > os.time()-86400 then
@@ -472,11 +475,7 @@
 		end
 
 		module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
-		if total_storage_usage then
-			module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
-		elseif total_storage_limit then
-			module:log("debug", "Global quota %s / %s", "not yet calculated", B(total_storage_limit));
-		end
+		module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
 
 		local obsolete_uploads = array();
 		local num_expired = 0;
@@ -513,11 +512,9 @@
 			deletion_query = {ids = obsolete_uploads};
 		end
 
-		if total_storage_usage then
-			total_storage_usage = total_storage_usage - size_sum;
-			module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
-			persist_stats:set(nil, "total", total_storage_usage);
-		end
+		total_storage_usage = total_storage_usage - size_sum;
+		module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
+		persist_stats:set(nil, "total", total_storage_usage);
 
 		if #obsolete_uploads == 0 then
 			module:log("debug", "No metadata to remove");
@@ -535,27 +532,27 @@
 	end);
 end
 
-if total_storage_limit then
-	local summary_start = module:measure("summary", "times");
+local summary_start = module:measure("summary", "times");
 
-	module:weekly("Global quota check", function()
-		local summary_done = summary_start();
-		local iter = assert(uploads:find(nil));
+module:weekly("Calculate total storage usage", function()
+	local summary_done = summary_start();
+	local iter = assert(uploads:find(nil));
 
-		local count, sum = 0, 0;
-		for _, file in iter do
-			sum = sum + tonumber(file.attr.size);
-			count = count + 1;
-		end
+	local count, sum = 0, 0;
+	for _, file in iter do
+		sum = sum + tonumber(file.attr.size);
+		count = count + 1;
+	end
 
-		module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+	module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+	if persist_stats:set(nil, "total", sum) then
 		total_storage_usage = sum;
-		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
-		persist_stats:set(nil, "total", sum);
-		summary_done();
-	end);
-
-end
+	else
+		total_storage_usage = unknown;
+	end
+	module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
+	summary_done();
+end);
 
 -- Reachable from the console
 function check_files(query)