mod_storage_s3/mod_storage_s3.lua
author Kim Alvefur <zash@zash.se>
Sat, 18 May 2024 14:16:49 +0200
changeset 5913 070b0db6c4a0
parent 5766 ff8b6d0b3bfa
permissions -rw-r--r--
mod_http_upload_external: Add link to Rust implementation (Thanks Luna)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     1
local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     2
local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     3
local async = require "prosody.util.async";
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
     4
local dt = require "prosody.util.datetime";
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     5
local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     6
local httputil = require "prosody.util.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     7
local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     8
local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
     9
local json = require "prosody.util.json";
5701
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
    10
local promise = require "prosody.util.promise";
5737
b6518a71ca7e mod_storage_s3: Implement search for set of IDs
Kim Alvefur <zash@zash.se>
parents: 5736
diff changeset
    11
local set = require "prosody.util.set";
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    12
local st = require "prosody.util.stanza";
5702
991fb904fa49 mod_storage_s3: Sort imports
Kim Alvefur <zash@zash.se>
parents: 5701
diff changeset
    13
local uuid = require "prosody.util.uuid";
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    14
local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    15
local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    16
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
    17
local new_uuid = uuid.v7 or uuid.generate;
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    18
local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    19
local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    20
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    21
local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    22
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    23
local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    24
local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    25
local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    26
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    27
local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    28
local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    29
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    30
local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    31
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    32
local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    33
	local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    34
	local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    35
	local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    36
	local payload = options.body;
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    37
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    38
	local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    39
	if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    40
		payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    41
		payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    42
	elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    43
		payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    44
		payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    45
	end
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
    46
	options.body = payload;
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    47
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    48
	local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    49
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    50
	local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    51
	local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    52
	local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    53
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    54
	local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    55
		["Accept"] = "*/*";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    56
		["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    57
		["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    58
		["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    59
		["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    60
		["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    61
		["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    62
	};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    63
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    64
	local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    65
	local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    66
	local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    67
	local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    68
5674
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    69
	if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    70
		local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    71
		for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    72
			sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    73
		end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    74
		sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    75
		canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    76
		request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    77
	end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5673
diff changeset
    78
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    79
	for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    80
		header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    81
		canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    82
		signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    83
	end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    84
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    85
	canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    86
	signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    87
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    88
	local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    89
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    90
	local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    91
		.. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    92
		.. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    93
		.. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    94
		.. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    95
		.. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    96
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    97
	local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    98
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
    99
	-- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   100
	local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   101
	local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   102
	local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   103
	local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   104
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   105
	local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   106
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   107
	headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   108
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   109
	options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   110
end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   111
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   112
function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   113
	local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   114
	if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   115
		return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   116
	end
5704
9de7a1b36efb mod_storage_s3: Enable connection pooling added in latest trunk
Kim Alvefur <zash@zash.se>
parents: 5703
diff changeset
   117
	local httpclient = http.new({ connection_pooling = true });
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   118
	httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   119
	return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   120
end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   121
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   122
local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   123
driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   124
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   125
local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   126
	local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   127
	request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   128
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   129
	return self.http:request(url.build(request), { method = method; body = payload; query = query });
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   130
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   131
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   132
-- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   133
local function on_result(response)
5680
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5679
diff changeset
   134
	if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5679
diff changeset
   135
		return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5679
diff changeset
   136
	end
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   137
	if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   138
		error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   139
	end
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   140
	local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   141
	if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   142
		return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   143
	elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   144
		return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   145
	elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   146
		return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   147
	else
5703
83a2fb6df746 mod_storage_s3: Fix logging
Kim Alvefur <zash@zash.se>
parents: 5702
diff changeset
   148
		module:log("warn", "Unknown response data type %s", content_type);
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   149
		return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   150
	end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   151
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   152
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   153
function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   154
	return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   155
		is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   156
		bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   157
		jid.escape(module.host);
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   158
		jid.escape(key or "@");
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   159
		jid.escape(self.store);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   160
	})
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   161
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   162
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   163
function keyval:get(user)
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   164
	return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   165
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   166
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   167
function keyval:set(user, data)
5675
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   168
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   169
	if data == nil or (type(data) == "table" and next(data) == nil) then
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   170
		return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5675
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   171
	end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5674
diff changeset
   172
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   173
	return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   174
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   175
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   176
function keyval:users()
5676
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   177
	local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
5762
32bc648e3892 mod_storage_s3: Fix passing of prefixes, should not be urlencoded
Kim Alvefur <zash@zash.se>
parents: 5741
diff changeset
   178
	local prefix = jid.escape(module.host) .. "/";
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   179
	local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5676
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   180
	if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   181
		return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   182
	end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   183
	local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   184
	if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   185
		local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   186
		module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   187
	end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   188
	local keys = array();
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   189
	local store_part = jid.escape(self.store);
5676
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   190
	for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   191
		local key = url.parse_path(content:get_child_text("Key"));
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   192
		if key[3] == store_part then
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   193
			keys:push(jid.unescape(key[2]));
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   194
		end
5676
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   195
	end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   196
	return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   197
		return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5675
diff changeset
   198
	end
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   199
end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   200
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   201
local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   202
driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   203
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   204
archive.caps = {
5736
e8938a3166d2 mod_storage_s3: Advertise full id range archive query capability
Kim Alvefur <zash@zash.se>
parents: 5732
diff changeset
   205
	full_id_range = true; -- both before and after used
5737
b6518a71ca7e mod_storage_s3: Implement search for set of IDs
Kim Alvefur <zash@zash.se>
parents: 5736
diff changeset
   206
	ids = true;
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   207
};
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   208
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   209
function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   210
	return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   211
		is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   212
		bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   213
		jid.escape(module.host);
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   214
		jid.escape(username or "@");
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   215
		jid.escape(self.store);
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   216
		date or dt.date(when);
5732
80702e33ba71 mod_storage_s3: Fix storing archives for host itself (e.g. mod_audit)
Kim Alvefur <zash@zash.se>
parents: 5704
diff changeset
   217
		jid.escape(with and jid.prep(with) or "@");
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   218
		key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   219
	})
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   220
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   221
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   222
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   223
-- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   224
function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   225
	key = key or new_uuid();
5764
59e38aaa3ec1 mod_storage_s3: Remove wrapper and original timestamp from payload (BC)
Kim Alvefur <zash@zash.se>
parents: 5763
diff changeset
   226
	return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, value):next(function(r)
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   227
		if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   228
			return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   229
		else
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   230
			error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   231
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   232
	end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   233
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   234
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   235
function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   236
	local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
5762
32bc648e3892 mod_storage_s3: Fix passing of prefixes, should not be urlencoded
Kim Alvefur <zash@zash.se>
parents: 5741
diff changeset
   237
	local prefix = { jid.escape(module.host); jid.escape(username or "@"); jid.escape(self.store) };
5678
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5677
diff changeset
   238
	if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5677
diff changeset
   239
		query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5677
diff changeset
   240
	end
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   241
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   242
	if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   243
		table.insert(prefix, dt.date(query["start"]));
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   244
		if query["with"] then
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   245
			table.insert(prefix, jid.escape(query["with"]));
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   246
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   247
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   248
5762
32bc648e3892 mod_storage_s3: Fix passing of prefixes, should not be urlencoded
Kim Alvefur <zash@zash.se>
parents: 5741
diff changeset
   249
	prefix = table.concat(prefix, "/").."/";
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   250
	local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   251
		prefix = prefix;
5766
ff8b6d0b3bfa mod_storage_s3: Fix mapping archive query limit to ?max-keys=
Kim Alvefur <zash@zash.se>
parents: 5764
diff changeset
   252
		["max-keys"] = query["limit"] and tostring(query["limit"]);
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   253
	}));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   254
	if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   255
		return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   256
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   257
	local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   258
	if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   259
		local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   260
		module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   261
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   262
	local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   263
	local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   264
		return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   265
	end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   266
	if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   267
		query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   268
		iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   269
	end
5737
b6518a71ca7e mod_storage_s3: Implement search for set of IDs
Kim Alvefur <zash@zash.se>
parents: 5736
diff changeset
   270
	local ids = query["ids"] and set.new(query["ids"]);
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   271
	local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   272
	for content in iterwrap(list_bucket_result:childtags("Contents")) do
5741
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   273
		local date, with, id = table.unpack(url.parse_path(content:get_child_text("Key")), 4);
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   274
		local when = dt.parse(content:get_child_text("LastModified"));
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   275
		with = jid.unescape(with);
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   276
		if found and query["before"] == id then
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   277
			break
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   278
		end
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   279
		if (not query["with"] or query["with"] == with)
5741
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   280
		and (not query["start"] or query["start"] <= when)
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   281
		and (not query["end"] or query["end"] >= when)
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   282
		and (not ids or ids:contains(id))
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   283
		and found then
5741
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   284
			keys:push({ key = id; date = date; when = when; with = with });
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   285
		end
5740
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents: 5739
diff changeset
   286
		if not found and id == query["after"] then
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   287
			found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   288
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   289
	end
5741
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   290
	keys:sort(function(a, b)
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   291
		if a.date ~= b.date then
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   292
			return a.date < b.date
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   293
		end
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   294
		if a.when ~= b.when then
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   295
			return a.when < b.when;
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   296
		end
5763
c7affea8bb24 mod_storage_s3: Fix sorting items by correct field
Kim Alvefur <zash@zash.se>
parents: 5762
diff changeset
   297
		return a.key < b.key;
5741
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   298
	end);
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   299
	if query["reverse"] then
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   300
		keys:reverse();
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   301
	end
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   302
	local i = 0;
5700
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   303
	local function get_next()
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   304
		i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   305
		local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   306
		if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   307
			return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   308
		end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   309
		-- luacheck: ignore 431/err
5699
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5680
diff changeset
   310
		local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   311
		if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   312
			module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   313
			return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   314
		end
5741
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
   315
		return item.key, value, item.when, item.with;
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   316
	end
5700
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
   317
	return get_next;
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   318
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   319
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   320
function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   321
	return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   322
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   323
5701
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   324
local function count(t) local n = 0; for _ in pairs(t) do n = n + 1; end return n; end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   325
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   326
function archive:delete(username, query)
5701
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   327
	local deletions = {};
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   328
	for key, _, when, with in self:find(username, query) do
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   329
		deletions[key] = new_request(self, "DELETE", self:_path(username or "@", dt.date(when), nil, with, key));
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   330
	end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents: 5700
diff changeset
   331
	return async.wait_for(promise.all(deletions):next(count));
5677
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   332
end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5676
diff changeset
   333
5673
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
   334
module:provides("storage", driver);