18 local dt = require "util.datetime"; |
18 local dt = require "util.datetime"; |
19 local hi = require "util.human.units"; |
19 local hi = require "util.human.units"; |
20 local cache = require "util.cache"; |
20 local cache = require "util.cache"; |
21 local lfs = require "lfs"; |
21 local lfs = require "lfs"; |
22 |
22 |
|
23 local unknown = math.abs(0/0); |
|
24 local unlimited = math.huge; |
|
25 |
23 local namespace = "urn:xmpp:http:upload:0"; |
26 local namespace = "urn:xmpp:http:upload:0"; |
24 |
27 |
25 module:depends("disco"); |
28 module:depends("disco"); |
26 |
29 |
27 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload")); |
30 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload")); |
36 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB |
39 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB |
37 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); |
40 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); |
38 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); |
41 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); |
39 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400); |
42 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400); |
40 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day |
43 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day |
41 local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil); |
44 local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited); |
42 |
45 |
43 local access = module:get_option_set(module.name .. "_access", {}); |
46 local access = module:get_option_set(module.name .. "_access", {}); |
44 |
47 |
45 if not external_base_url then |
48 if not external_base_url then |
46 module:depends("http"); |
49 module:depends("http"); |
58 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; |
61 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; |
59 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; |
62 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; |
60 }; |
63 }; |
61 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; |
64 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; |
62 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; |
65 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; |
63 unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" }; |
|
64 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; |
66 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; |
65 }); |
67 }); |
66 |
68 |
67 local upload_cache = cache.new(1024); |
69 local upload_cache = cache.new(1024); |
68 local quota_cache = cache.new(1024); |
70 local quota_cache = cache.new(1024); |
69 |
71 |
70 local total_storage_usage = nil; |
72 local total_storage_usage = unknown; |
71 |
73 |
72 local measure_upload_cache_size = module:measure("upload_cache", "amount"); |
74 local measure_upload_cache_size = module:measure("upload_cache", "amount"); |
73 local measure_quota_cache_size = module:measure("quota_cache", "amount"); |
75 local measure_quota_cache_size = module:measure("quota_cache", "amount"); |
74 local measure_total_storage_usage = nil; |
76 local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" }); |
75 if total_storage_limit then |
77 |
|
78 do |
76 local total, err = persist_stats:get(nil, "total"); |
79 local total, err = persist_stats:get(nil, "total"); |
77 if not err then total_storage_usage = tonumber(total) or 0; end |
80 if not err then |
78 measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" }); |
81 total_storage_usage = tonumber(total) or 0; |
|
82 end |
79 end |
83 end |
80 |
84 |
81 module:hook_global("stats-update", function () |
85 module:hook_global("stats-update", function () |
82 measure_upload_cache_size(upload_cache:count()); |
86 measure_upload_cache_size(upload_cache:count()); |
83 measure_quota_cache_size(quota_cache:count()); |
87 measure_quota_cache_size(quota_cache:count()); |
84 if total_storage_limit and total_storage_usage then |
88 measure_total_storage_usage(total_storage_usage); |
85 measure_total_storage_usage(total_storage_usage); |
|
86 end |
|
87 end); |
89 end); |
88 |
90 |
89 local buckets = {}; |
91 local buckets = {}; |
90 for n = 10, 40, 2 do |
92 for n = 10, 40, 2 do |
91 local exp = math.floor(2 ^ n); |
93 local exp = math.floor(2 ^ n); |
93 if exp >= file_size_limit then break end |
95 if exp >= file_size_limit then break end |
94 end |
96 end |
95 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets}); |
97 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets}); |
96 |
98 |
97 -- Convenience wrapper for logging file sizes |
99 -- Convenience wrapper for logging file sizes |
98 local function B(bytes) return hi.format(bytes, "B", "b"); end |
100 local function B(bytes) |
|
101 if bytes ~= bytes then |
|
102 return "unknown" |
|
103 elseif bytes == unlimited then |
|
104 return "unlimited"; |
|
105 end |
|
106 return hi.format(bytes, "B", "b"); |
|
107 end |
99 |
108 |
100 local function get_filename(slot, create) |
109 local function get_filename(slot, create) |
101 return dm.getpath(slot, module.host, module.name, "bin", create) |
110 return dm.getpath(slot, module.host, module.name, "bin", create) |
102 end |
111 end |
103 |
112 |
139 end |
148 end |
140 if filesize > file_size_limit then |
149 if filesize > file_size_limit then |
141 return false, upload_errors.new("filesize"); |
150 return false, upload_errors.new("filesize"); |
142 end |
151 end |
143 |
152 |
144 if total_storage_limit then |
153 if total_storage_usage + filesize > total_storage_limit then |
145 if not total_storage_usage then |
154 module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit)); |
146 return false, upload_errors.new("unknowntotal"); |
155 return false, upload_errors.new("outofdisk"); |
147 elseif total_storage_usage + filesize > total_storage_limit then |
|
148 module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage)); |
|
149 return false, upload_errors.new("outofdisk"); |
|
150 end |
|
151 end |
156 end |
152 |
157 |
153 local uploader_quota = get_daily_quota(uploader); |
158 local uploader_quota = get_daily_quota(uploader); |
154 if uploader_quota + filesize > daily_quota then |
159 if uploader_quota + filesize > daily_quota then |
155 return false, upload_errors.new("quota"); |
160 return false, upload_errors.new("quota"); |
215 if not slot then |
220 if not slot then |
216 origin.send(st.error_reply(stanza, storage_err)); |
221 origin.send(st.error_reply(stanza, storage_err)); |
217 return true; |
222 return true; |
218 end |
223 end |
219 |
224 |
220 if total_storage_usage then |
225 total_storage_usage = total_storage_usage + filesize; |
221 total_storage_usage = total_storage_usage + filesize; |
226 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
222 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); |
|
223 end |
|
224 |
227 |
225 local cached_quota = quota_cache:get(uploader); |
228 local cached_quota = quota_cache:get(uploader); |
226 if cached_quota and cached_quota.time > os.time()-86400 then |
229 if cached_quota and cached_quota.time > os.time()-86400 then |
227 cached_quota.size = cached_quota.size + filesize; |
230 cached_quota.size = cached_quota.size + filesize; |
228 quota_cache:set(uploader, cached_quota); |
231 quota_cache:set(uploader, cached_quota); |
470 prune_done(); |
473 prune_done(); |
471 return; |
474 return; |
472 end |
475 end |
473 |
476 |
474 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); |
477 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); |
475 if total_storage_usage then |
478 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
476 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); |
|
477 elseif total_storage_limit then |
|
478 module:log("debug", "Global quota %s / %s", "not yet calculated", B(total_storage_limit)); |
|
479 end |
|
480 |
479 |
481 local obsolete_uploads = array(); |
480 local obsolete_uploads = array(); |
482 local num_expired = 0; |
481 local num_expired = 0; |
483 local size_sum = 0; |
482 local size_sum = 0; |
484 local problem_deleting = false; |
483 local problem_deleting = false; |
511 -- eventually the admin ought to notice and fix the permissions or |
510 -- eventually the admin ought to notice and fix the permissions or |
512 -- whatever the problem is. |
511 -- whatever the problem is. |
513 deletion_query = {ids = obsolete_uploads}; |
512 deletion_query = {ids = obsolete_uploads}; |
514 end |
513 end |
515 |
514 |
516 if total_storage_usage then |
515 total_storage_usage = total_storage_usage - size_sum; |
517 total_storage_usage = total_storage_usage - size_sum; |
516 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
518 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); |
517 persist_stats:set(nil, "total", total_storage_usage); |
519 persist_stats:set(nil, "total", total_storage_usage); |
|
520 end |
|
521 |
518 |
522 if #obsolete_uploads == 0 then |
519 if #obsolete_uploads == 0 then |
523 module:log("debug", "No metadata to remove"); |
520 module:log("debug", "No metadata to remove"); |
524 else |
521 else |
525 local removed, err = uploads:delete(nil, deletion_query); |
522 local removed, err = uploads:delete(nil, deletion_query); |
533 |
530 |
534 prune_done(); |
531 prune_done(); |
535 end); |
532 end); |
536 end |
533 end |
537 |
534 |
538 if total_storage_limit then |
535 local summary_start = module:measure("summary", "times"); |
539 local summary_start = module:measure("summary", "times"); |
536 |
540 |
537 module:weekly("Calculate total storage usage", function() |
541 module:weekly("Global quota check", function() |
538 local summary_done = summary_start(); |
542 local summary_done = summary_start(); |
539 local iter = assert(uploads:find(nil)); |
543 local iter = assert(uploads:find(nil)); |
540 |
544 |
541 local count, sum = 0, 0; |
545 local count, sum = 0, 0; |
542 for _, file in iter do |
546 for _, file in iter do |
543 sum = sum + tonumber(file.attr.size); |
547 sum = sum + tonumber(file.attr.size); |
544 count = count + 1; |
548 count = count + 1; |
545 end |
549 end |
546 |
550 |
547 module:log("info", "Uploaded files total: %s in %d files", B(sum), count); |
551 module:log("info", "Uploaded files total: %s in %d files", B(sum), count); |
548 if persist_stats:set(nil, "total", sum) then |
552 total_storage_usage = sum; |
549 total_storage_usage = sum; |
553 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); |
550 else |
554 persist_stats:set(nil, "total", sum); |
551 total_storage_usage = unknown; |
555 summary_done(); |
552 end |
556 end); |
553 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
557 |
554 summary_done(); |
558 end |
555 end); |
559 |
556 |
560 -- Reachable from the console |
557 -- Reachable from the console |
561 function check_files(query) |
558 function check_files(query) |
562 local issues = {}; |
559 local issues = {}; |
563 local iter = assert(uploads:find(nil, query)); |
560 local iter = assert(uploads:find(nil, query)); |