mod_storage_s3/mod_storage_s3.lua
author Kim Alvefur <zash@zash.se>
Sat, 11 Nov 2023 22:26:39 +0100
changeset 5700 66986f5271c3
parent 5699 b4632d5f840b
child 5701 4a0279c5c7ed
permissions -rw-r--r--
mod_storage_s3: Skip archive items matching on date but not full datetime Since it only encodes dates in paths, it would have returned items from outside the specified start..end range if they were from earlier or later in the same (UTC) day.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     3
local async = require "prosody.util.async";
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
     4
local dt = require "prosody.util.datetime";
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
local httputil = require "prosody.util.http";
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
     7
local uuid = require "prosody.util.uuid";
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     8
local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     9
local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    10
local json = require "prosody.util.json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    11
local st = require "prosody.util.stanza";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    12
local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    13
local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    14
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
    15
local new_uuid = uuid.v7 or uuid.generate;
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    16
local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    17
local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    18
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    19
local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    20
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    21
local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    22
local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    23
local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    24
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    25
local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    26
local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    27
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    28
local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    29
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    30
local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    31
	local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    32
	local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    33
	local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    34
	local payload = options.body;
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    35
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    36
	local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    37
	if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    38
		payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    39
		payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    40
	elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    41
		payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    42
		payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    43
	end
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    44
	options.body = payload;
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    45
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    46
	local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    47
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    48
	local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    49
	local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    50
	local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    51
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    52
	local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    53
		["Accept"] = "*/*";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    54
		["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    55
		["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    56
		["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    57
		["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    58
		["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    59
		["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    60
	};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    61
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    62
	local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    63
	local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    64
	local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    65
	local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    66
5674
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    67
	if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    68
		local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    69
		for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    70
			sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    71
		end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    72
		sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    73
		canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    74
		request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    75
	end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    76
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    77
	for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    78
		header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    79
		canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    80
		signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
	end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    82
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    83
	canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    84
	signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    85
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    86
	local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    87
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    88
	local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    89
		.. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    90
		.. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    91
		.. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    92
		.. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    93
		.. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    94
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    95
	local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    96
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    97
	-- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    98
	local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    99
	local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   100
	local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   101
	local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   102
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   103
	local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   104
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   105
	headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   106
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   107
	options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   108
end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   109
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   110
function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   111
	local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   112
	if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   113
		return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   114
	end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   115
	local httpclient = http.new({});
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   116
	httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   117
	return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   118
end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   119
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   120
local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   121
driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   122
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   123
local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   124
	local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   125
	request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   126
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   127
	return self.http:request(url.build(request), { method = method; body = payload; query = query });
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   128
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   129
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   130
-- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   131
local function on_result(response)
5680
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5679
diff changeset
   132
	if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5679
diff changeset
   133
		return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5679
diff changeset
   134
	end
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   135
	if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   136
		error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   137
	end
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   138
	local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   139
	if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   140
		return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   141
	elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   142
		return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   143
	elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   144
		return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   145
	else
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   146
		response.log("warn", "Unknown response data type %s", content_type);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   147
		return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   148
	end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   149
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   150
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   151
function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   152
	return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   153
		is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   154
		bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   155
		jid.escape(module.host);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   156
		jid.escape(self.store);
5679
17ea26cf7259 mod_storage_s3: Use '@' as placeholder for empty (host) store slots
Kim Alvefur <zash@zash.se>
parents: 5678
diff changeset
   157
		jid.escape(key or "@");
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   158
	})
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   159
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   160
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   161
function keyval:get(user)
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   162
	return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   163
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   164
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   165
function keyval:set(user, data)
5675
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   166
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   167
	if data == nil or (type(data) == "table" and next(data) == nil) then
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   168
		return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5675
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   169
	end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   170
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   171
	return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   172
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   173
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   174
function keyval:users()
5676
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   175
	local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   176
	local prefix = url.build_path({ jid.escape(module.host); jid.escape(self.store); is_directory = true });
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   177
	local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5676
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   178
	if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   179
		return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   180
	end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   181
	local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   182
	if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   183
		local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   184
		module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   185
	end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   186
	local keys = array();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   187
	for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   188
		local key = url.parse_path(content:get_child_text("Key"));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   189
		keys:push(jid.unescape(key[3]));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   190
	end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   191
	return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   192
		return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   193
	end
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   194
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   195
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   196
local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   197
driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   198
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   199
archive.caps = {
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   200
};
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   201
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   202
function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   203
	return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   204
		is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   205
		bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   206
		jid.escape(module.host);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   207
		jid.escape(self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   208
		jid.escape(username);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   209
		jid.escape(jid.prep(with));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   210
		date or dt.date(when);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   211
		key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   212
	})
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   213
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   214
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   215
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   216
-- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   217
function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   218
	local wrapper = st.stanza("wrapper");
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   219
	-- Minio had trouble with timestamps, probably the ':' characters, in paths.
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   220
	wrapper:tag("delay", { xmlns = "urn:xmpp:delay"; stamp = dt.datetime(when) }):up();
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   221
	wrapper:add_direct_child(value);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   222
	key = key or new_uuid();
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   223
	return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, wrapper):next(function(r)
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   224
		if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   225
			return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   226
		else
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   227
			error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   228
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   229
	end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   230
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   231
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   232
function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   233
	local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   234
	local prefix = { jid.escape(module.host); jid.escape(self.store); is_directory = true };
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   235
	table.insert(prefix, jid.escape(username or "@"));
5678
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5677
diff changeset
   236
	if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5677
diff changeset
   237
		query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5677
diff changeset
   238
	end
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   239
	if query["with"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   240
		table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   241
		if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   242
			table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   243
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   244
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   245
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   246
	prefix = url.build_path(prefix);
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   247
	local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   248
		prefix = prefix;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   249
		["max-keys"] = query["max"] and tostring(query["max"]);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   250
	}));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   251
	if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   252
		return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   253
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   254
	local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   255
	if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   256
		local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   257
		module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   258
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   259
	local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   260
	local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   261
		return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   262
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   263
	if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   264
		query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   265
		iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   266
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   267
	local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   268
	for content in iterwrap(list_bucket_result:childtags("Contents")) do
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   269
		local key = url.parse_path(content:get_child_text("Key"));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   270
		if found and query["before"] == key[6] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   271
			break
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   272
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   273
		if (not query["with"] or query["with"] == jid.unescape(key[5]))
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   274
		and (not query["start"] or dt.date(query["start"]) >= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   275
		and (not query["end"] or dt.date(query["end"]) <= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   276
		and found then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   277
			keys:push({ key = key[6]; date = key[5]; with = jid.unescape(key[4]) });
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   278
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   279
		if not found and key[6] == query["after"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   280
			found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   281
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   282
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   283
	local i = 0;
5700
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   284
	local function get_next()
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   285
		i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   286
		local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   287
		if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   288
			return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   289
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   290
		-- luacheck: ignore 431/err
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   291
		local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   292
		if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   293
			module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   294
			return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   295
		end
5700
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   296
		local when = dt.parse(value:get_child_attr("delay", "urn:xmpp:delay", "stamp"));
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   297
5700
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   298
		if (not query["start"] or query["start"] >= when) and (not query["end"] or query["end"] <= when) then
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   299
			return item.key, value.tags[2], when, item.with;
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   300
		else
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   301
			-- date was correct but not the time
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   302
			return get_next();
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   303
		end
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   304
	end
5700
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   305
	return get_next;
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   306
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   307
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   308
function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   309
	return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   310
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   311
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   312
--[[ TODO
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   313
function archive:delete(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   314
	return nil, "not-implemented";
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   315
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   316
--]]
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   317
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   318
module:provides("storage", driver);