author | Matthew Wild <mwild1@gmail.com> |
Tue, 16 Apr 2024 12:58:08 +0100 | |
changeset 5889 | 54b451c3790c |
parent 5766 | ff8b6d0b3bfa |
permissions | -rw-r--r-- |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 |
local http = require "prosody.net.http"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 |
local array = require "prosody.util.array"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 |
local async = require "prosody.util.async"; |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
4 |
local dt = require "prosody.util.datetime"; |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 |
local hashes = require "prosody.util.hashes"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 |
local httputil = require "prosody.util.http"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 |
local it = require "prosody.util.iterators"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 |
local jid = require "prosody.util.jid"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 |
local json = require "prosody.util.json"; |
5701
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
10 |
local promise = require "prosody.util.promise"; |
5737
b6518a71ca7e
mod_storage_s3: Implement search for set of IDs
Kim Alvefur <zash@zash.se>
parents:
5736
diff
changeset
|
11 |
local set = require "prosody.util.set"; |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
12 |
local st = require "prosody.util.stanza"; |
5702 | 13 |
local uuid = require "prosody.util.uuid"; |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
14 |
local xml = require "prosody.util.xml"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 |
local url = require "socket.url"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
16 |
|
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
17 |
local new_uuid = uuid.v7 or uuid.generate; |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
18 |
local hmac_sha256 = hashes.hmac_sha256; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
19 |
local sha256 = hashes.sha256; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
20 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
21 |
local driver = {}; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
22 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
23 |
local bucket = module:get_option_string("s3_bucket", "prosody"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
24 |
local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
25 |
local region = module:get_option_string("s3_region", "us-east-1"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
26 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 |
local access_key = module:get_option_string("s3_access_key"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 |
local secret_key = module:get_option_string("s3_secret_key"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 |
local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
31 |
|
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
32 |
local function aws_auth(event) |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
33 |
local request, options = event.request, event.options; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
34 |
local method = options.method or "GET"; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
35 |
local query = options.query; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
36 |
local payload = options.body; |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
38 |
local payload_type = nil; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
39 |
if st.is_stanza(payload) then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
40 |
payload_type = "application/xml"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
41 |
payload = tostring(payload); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
42 |
elseif payload ~= nil then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
43 |
payload_type = "application/json"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
44 |
payload = json.encode(payload); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
45 |
end |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
46 |
options.body = payload; |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
47 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
48 |
local payload_hash = sha256(payload or "", true); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
49 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
50 |
local now = os.time(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
51 |
local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
52 |
local aws_date = os.date("!%Y%m%d", now); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
53 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
54 |
local headers = { |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
55 |
["Accept"] = "*/*"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
56 |
["Authorization"] = nil; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
57 |
["Content-Type"] = payload_type; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
58 |
["Host"] = request.authority; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
59 |
["User-Agent"] = "Prosody XMPP Server"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
60 |
["X-Amz-Content-Sha256"] = payload_hash; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
61 |
["X-Amz-Date"] = aws_datetime; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
62 |
}; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
63 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
64 |
local canonical_uri = url.build({ path = request.path }); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
65 |
local canonical_query = ""; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
66 |
local canonical_headers = array(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
67 |
local signed_headers = array() |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
68 |
|
5674
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
69 |
if query then |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
70 |
local sorted_query = array(); |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
71 |
for name, value in it.sorted_pairs(query) do |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
72 |
sorted_query:push({ name = name; value = value }); |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
73 |
end |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
74 |
sorted_query:sort(function (a,b) return a.name < b.name end) |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
75 |
canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper); |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
76 |
request.query = canonical_query; |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
77 |
end |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
78 |
|
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
79 |
for header_name, header_value in it.sorted_pairs(headers) do |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
80 |
header_name = header_name:lower(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
81 |
canonical_headers:push(header_name .. ":" .. header_value .. "\n"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
82 |
signed_headers:push(header_name); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
83 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
84 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
85 |
canonical_headers = canonical_headers:concat(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
86 |
signed_headers = signed_headers:concat(";"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
87 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
88 |
local scope = aws_date .. "/" .. region .. "/s3/aws4_request"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
89 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
90 |
local canonical_request = method .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
91 |
.. canonical_uri .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
92 |
.. canonical_query .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
93 |
.. canonical_headers .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
94 |
.. signed_headers .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
95 |
.. payload_hash; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
96 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
97 |
local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
98 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
99 |
-- This can be cached? |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
100 |
local date_key = hmac_sha256("AWS4" .. secret_key, aws_date); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
101 |
local date_region_key = hmac_sha256(date_key, region); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
102 |
local date_region_service_key = hmac_sha256(date_region_key, "s3"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
103 |
local signing_key = hmac_sha256(date_region_service_key, "aws4_request"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
104 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
105 |
local signature = hmac_sha256(signing_key, signature_payload, true); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
106 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
107 |
headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
108 |
|
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
109 |
options.headers = headers; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
110 |
end |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
111 |
|
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
112 |
function driver:open(store, typ) |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
113 |
local mt = self[typ or "keyval"] |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
114 |
if not mt then |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
115 |
return nil, "unsupported-store"; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
116 |
end |
5704
9de7a1b36efb
mod_storage_s3: Enable connection pooling added in latest trunk
Kim Alvefur <zash@zash.se>
parents:
5703
diff
changeset
|
117 |
local httpclient = http.new({ connection_pooling = true }); |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
118 |
httpclient.events.add_handler("pre-request", aws_auth); |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
119 |
return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt); |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
120 |
end |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
121 |
|
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
122 |
local keyval = { }; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
123 |
driver.keyval = { __index = keyval; __name = module.name .. " keyval store" }; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
124 |
|
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
125 |
local function new_request(self, method, path, query, payload) |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
126 |
local request = url.parse(base_uri); |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
127 |
request.path = path; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
128 |
|
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
129 |
return self.http:request(url.build(request), { method = method; body = payload; query = query }); |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
130 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
131 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
132 |
-- coerce result back into Prosody data type |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
133 |
local function on_result(response) |
5680
799f69a5921a
mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents:
5679
diff
changeset
|
134 |
if response.code == 404 and response.request.method == "GET" then |
799f69a5921a
mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents:
5679
diff
changeset
|
135 |
return nil; |
799f69a5921a
mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents:
5679
diff
changeset
|
136 |
end |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
137 |
if response.code >= 400 then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
138 |
error(response.body); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
139 |
end |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
140 |
local content_type = response.headers["content-type"]; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
141 |
if content_type == "application/json" then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
142 |
return json.decode(response.body); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
143 |
elseif content_type == "application/xml" then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
144 |
return xml.parse(response.body); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
145 |
elseif content_type == "application/x-www-form-urlencoded" then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
146 |
return httputil.formdecode(response.body); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
147 |
else |
5703 | 148 |
module:log("warn", "Unknown response data type %s", content_type); |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
149 |
return response.body; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
150 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
151 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
152 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
153 |
function keyval:_path(key) |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
154 |
return url.build_path({ |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
155 |
is_absolute = true; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
156 |
bucket; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
157 |
jid.escape(module.host); |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
158 |
jid.escape(key or "@"); |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
159 |
jid.escape(self.store); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
160 |
}) |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
161 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
162 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
163 |
function keyval:get(user) |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
164 |
return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result)); |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
165 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
166 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
167 |
function keyval:set(user, data) |
5675
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5674
diff
changeset
|
168 |
|
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5674
diff
changeset
|
169 |
if data == nil or (type(data) == "table" and next(data) == nil) then |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
170 |
return async.wait_for(new_request(self, "DELETE", self:_path(user))); |
5675
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5674
diff
changeset
|
171 |
end |
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5674
diff
changeset
|
172 |
|
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
173 |
return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data)); |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
174 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
175 |
|
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
176 |
function keyval:users() |
5676
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
177 |
local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true }); |
5762
32bc648e3892
mod_storage_s3: Fix passing of prefixes, should not be urlencoded
Kim Alvefur <zash@zash.se>
parents:
5741
diff
changeset
|
178 |
local prefix = jid.escape(module.host) .. "/"; |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
179 |
local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix })) |
5676
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
180 |
if err or list_result.code ~= 200 then |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
181 |
return nil, err; |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
182 |
end |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
183 |
local list_bucket_result = xml.parse(list_result.body); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
184 |
if list_bucket_result:get_child_text("IsTruncated") == "true" then |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
185 |
local max_keys = list_bucket_result:get_child_text("MaxKeys"); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
186 |
module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
187 |
end |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
188 |
local keys = array(); |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
189 |
local store_part = jid.escape(self.store); |
5676
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
190 |
for content in list_bucket_result:childtags("Contents") do |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
191 |
local key = url.parse_path(content:get_child_text("Key")); |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
192 |
if key[3] == store_part then |
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
193 |
keys:push(jid.unescape(key[2])); |
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
194 |
end |
5676
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
195 |
end |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
196 |
return function() |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
197 |
return keys:pop(); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
198 |
end |
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
199 |
end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
200 |
|
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
201 |
local archive = {}; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
202 |
driver.archive = { __index = archive }; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
203 |
|
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
204 |
archive.caps = { |
5736
e8938a3166d2
mod_storage_s3: Advertise full id range archive query capability
Kim Alvefur <zash@zash.se>
parents:
5732
diff
changeset
|
205 |
full_id_range = true; -- both before and after used |
5737
b6518a71ca7e
mod_storage_s3: Implement search for set of IDs
Kim Alvefur <zash@zash.se>
parents:
5736
diff
changeset
|
206 |
ids = true; |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
207 |
}; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
208 |
|
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
209 |
function archive:_path(username, date, when, with, key) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
210 |
return url.build_path({ |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
211 |
is_absolute = true; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
212 |
bucket; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
213 |
jid.escape(module.host); |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
214 |
jid.escape(username or "@"); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
215 |
jid.escape(self.store); |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
216 |
date or dt.date(when); |
5732
80702e33ba71
mod_storage_s3: Fix storing archives for host itself (e.g. mod_audit)
Kim Alvefur <zash@zash.se>
parents:
5704
diff
changeset
|
217 |
jid.escape(with and jid.prep(with) or "@"); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
218 |
key; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
219 |
}) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
220 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
221 |
|
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
222 |
|
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
223 |
-- PUT .../with/when/id |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
224 |
function archive:append(username, key, value, when, with) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
225 |
key = key or new_uuid(); |
5764
59e38aaa3ec1
mod_storage_s3: Remove wrapper and original timestamp from payload (BC)
Kim Alvefur <zash@zash.se>
parents:
5763
diff
changeset
|
226 |
return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, value):next(function(r) |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
227 |
if r.code == 200 then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
228 |
return key; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
229 |
else |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
230 |
error(r.body); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
231 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
232 |
end)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
233 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
234 |
|
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
235 |
function archive:find(username, query) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
236 |
local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true }); |
5762
32bc648e3892
mod_storage_s3: Fix passing of prefixes, should not be urlencoded
Kim Alvefur <zash@zash.se>
parents:
5741
diff
changeset
|
237 |
local prefix = { jid.escape(module.host); jid.escape(username or "@"); jid.escape(self.store) }; |
5678
51d0311747fa
mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents:
5677
diff
changeset
|
238 |
if not query then |
51d0311747fa
mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents:
5677
diff
changeset
|
239 |
query = {}; |
51d0311747fa
mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents:
5677
diff
changeset
|
240 |
end |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
241 |
|
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
242 |
if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then |
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
243 |
table.insert(prefix, dt.date(query["start"])); |
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
244 |
if query["with"] then |
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
245 |
table.insert(prefix, jid.escape(query["with"])); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
246 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
247 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
248 |
|
5762
32bc648e3892
mod_storage_s3: Fix passing of prefixes, should not be urlencoded
Kim Alvefur <zash@zash.se>
parents:
5741
diff
changeset
|
249 |
prefix = table.concat(prefix, "/").."/"; |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
250 |
local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
251 |
prefix = prefix; |
5766
ff8b6d0b3bfa
mod_storage_s3: Fix mapping archive query limit to ?max-keys=
Kim Alvefur <zash@zash.se>
parents:
5764
diff
changeset
|
252 |
["max-keys"] = query["limit"] and tostring(query["limit"]); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
253 |
})); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
254 |
if err or list_result.code ~= 200 then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
255 |
return nil, err; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
256 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
257 |
local list_bucket_result = xml.parse(list_result.body); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
258 |
if list_bucket_result:get_child_text("IsTruncated") == "true" then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
259 |
local max_keys = list_bucket_result:get_child_text("MaxKeys"); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
260 |
module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
261 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
262 |
local keys = array(); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
263 |
local iterwrap = function(...) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
264 |
return ...; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
265 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
266 |
if query["reverse"] then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
267 |
query["before"], query["after"] = query["after"], query["before"]; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
268 |
iterwrap = it.reverse; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
269 |
end |
5737
b6518a71ca7e
mod_storage_s3: Implement search for set of IDs
Kim Alvefur <zash@zash.se>
parents:
5736
diff
changeset
|
270 |
local ids = query["ids"] and set.new(query["ids"]); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
271 |
local found = not query["after"]; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
272 |
for content in iterwrap(list_bucket_result:childtags("Contents")) do |
5741
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
273 |
local date, with, id = table.unpack(url.parse_path(content:get_child_text("Key")), 4); |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
274 |
local when = dt.parse(content:get_child_text("LastModified")); |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
275 |
with = jid.unescape(with); |
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
276 |
if found and query["before"] == id then |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
277 |
break |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
278 |
end |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
279 |
if (not query["with"] or query["with"] == with) |
5741
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
280 |
and (not query["start"] or query["start"] <= when) |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
281 |
and (not query["end"] or query["end"] >= when) |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
282 |
and (not ids or ids:contains(id)) |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
283 |
and found then |
5741
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
284 |
keys:push({ key = id; date = date; when = when; with = with }); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
285 |
end |
5740
ba731ff5b895
mod_storage_s3: Reorder path components (BC: invalidates any existing data)
Kim Alvefur <zash@zash.se>
parents:
5739
diff
changeset
|
286 |
if not found and id == query["after"] then |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
287 |
found = not found |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
288 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
289 |
end |
5741
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
290 |
keys:sort(function(a, b) |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
291 |
if a.date ~= b.date then |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
292 |
return a.date < b.date |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
293 |
end |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
294 |
if a.when ~= b.when then |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
295 |
return a.when < b.when; |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
296 |
end |
5763
c7affea8bb24
mod_storage_s3: Fix sorting items by correct field
Kim Alvefur <zash@zash.se>
parents:
5762
diff
changeset
|
297 |
return a.key < b.key; |
5741
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
298 |
end); |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
299 |
if query["reverse"] then |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
300 |
keys:reverse(); |
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
301 |
end |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
302 |
local i = 0; |
5700
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5699
diff
changeset
|
303 |
local function get_next() |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
304 |
i = i + 1; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
305 |
local item = keys[i]; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
306 |
if item == nil then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
307 |
return nil; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
308 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
309 |
-- luacheck: ignore 431/err |
5699
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5680
diff
changeset
|
310 |
local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result)); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
311 |
if not value then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
312 |
module:log("error", "%s", err); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
313 |
return nil; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
314 |
end |
5741
72b0fa7e36dc
mod_storage_s3: Sort archive items by LastModified
Kim Alvefur <zash@zash.se>
parents:
5740
diff
changeset
|
315 |
return item.key, value, item.when, item.with; |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
316 |
end |
5700
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5699
diff
changeset
|
317 |
return get_next; |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
318 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
319 |
|
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
320 |
function archive:users() |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
321 |
return it.unique(keyval.users(self)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
322 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
323 |
|
5701
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
324 |
local function count(t) local n = 0; for _ in pairs(t) do n = n + 1; end return n; end |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
325 |
|
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
326 |
function archive:delete(username, query) |
5701
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
327 |
local deletions = {}; |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
328 |
for key, _, when, with in self:find(username, query) do |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
329 |
deletions[key] = new_request(self, "DELETE", self:_path(username or "@", dt.date(when), nil, with, key)); |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
330 |
end |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5700
diff
changeset
|
331 |
return async.wait_for(promise.all(deletions):next(count)); |
5677
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
332 |
end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
333 |
|
5673
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
334 |
module:provides("storage", driver); |