author | Kim Alvefur <zash@zash.se> |
Sun, 25 Jun 2023 16:45:25 +0200 | |
changeset 5578 | 10b7118bbd8f |
parent 5577 | ed177fa5b54c |
child 5579 | 372e59bd45d7 |
permissions | -rw-r--r-- |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 |
-- Fetches Atom feeds and publishes to PubSub nodes |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 |
-- |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 |
-- Config: |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 |
-- Component "pubsub.example.com" "pubsub" |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 |
-- modules_enabled = { |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
6 |
-- "pubsub_feeds"; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 |
-- } |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 |
-- feeds = { -- node -> url |
279
aa0df3db4901
mod_pubsub_feed: Wrap entry in a item element.
Kim Alvefur <zash@zash.se>
parents:
278
diff
changeset
|
9 |
-- prosody_blog = "http://blog.prosody.im/feed/atom.xml"; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 |
-- } |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
11 |
-- feed_pull_interval = 20 -- minutes |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
12 |
-- |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
13 |
-- Reference |
1454
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
14 |
-- http://pubsubhubbub.googlecode.com/svn/trunk/pubsubhubbub-core-0.4.html |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 |
|
792
db27f07342d4
mod_pubsub_feeds: Use reference to mod_pubsub from module:depends
Kim Alvefur <zash@zash.se>
parents:
765
diff
changeset
|
16 |
local pubsub = module:depends"pubsub"; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
17 |
|
3051
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
18 |
local time = os.time; |
1325
b21236b6b8d8
Backed out changeset 853a382c9bd6
Kim Alvefur <zash@zash.se>
parents:
1324
diff
changeset
|
19 |
local dt_parse, dt_datetime = require "util.datetime".parse, require "util.datetime".datetime; |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
20 |
local uuid = require "util.uuid".generate; |
1107
04cf0b75fe2a
mod_pubsub_feeds: Use HMAC-SHA1 from util.hashes (util.hmac was deprecated)
Kim Alvefur <zash@zash.se>
parents:
983
diff
changeset
|
21 |
local hmac_sha1 = require "util.hashes".hmac_sha1; |
2141
0fe725b50f77
mod_pubsub_feeds.lua: util.xml not uit.xml
Michael Töglhofer <michael@toeglhofer.net>
parents:
2138
diff
changeset
|
22 |
local parse_xml = require "util.xml".parse; |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
23 |
local st = require "util.stanza"; |
2136
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
24 |
local translate_rss = module:require("feeds").translate_rss; |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
25 |
|
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
26 |
local xmlns_atom = "http://www.w3.org/2005/Atom"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
27 |
|
2136
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
28 |
local function parse_feed(data) |
4887
68bd8ae9b827
mod_pubsub_feeds: Allow but ignore XML comments and processing instructions
Kim Alvefur <zash@zash.se>
parents:
4286
diff
changeset
|
29 |
local feed, err = parse_xml(data, { allow_processing_instructions = true; allow_comments = true }); |
2136
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
30 |
if not feed then return feed, err; end |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
31 |
if feed.attr.xmlns == xmlns_atom then |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
32 |
return feed; |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
33 |
elseif feed.attr.xmlns == nil and feed.name == "rss" then |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
34 |
return translate_rss(feed); |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
35 |
end |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
36 |
return nil, "unsupported-format"; |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
37 |
end |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
38 |
|
5573
1f8c9e27b625
mod_pubsub_feeds: Disable WebSub (formerly PubSubHubbub) by default
Kim Alvefur <zash@zash.se>
parents:
4887
diff
changeset
|
39 |
local use_pubsubhubub = module:get_option_boolean("use_pubsubhubub", false); |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
40 |
if use_pubsubhubub then |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
41 |
module:depends"http"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
42 |
end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
43 |
|
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
44 |
local http = require "net.http"; |
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
45 |
local formdecode = http.formdecode; |
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
46 |
local formencode = http.formencode; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
47 |
|
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
48 |
local feed_list = module:shared("feed_list"); |
5574
f93b1fc1aa31
mod_pubsub_feeds: Add new interval setting in seconds (old still works)
Kim Alvefur <zash@zash.se>
parents:
5573
diff
changeset
|
49 |
local legacy_refresh_interval = module:get_option_number("feed_pull_interval", 15); |
f93b1fc1aa31
mod_pubsub_feeds: Add new interval setting in seconds (old still works)
Kim Alvefur <zash@zash.se>
parents:
5573
diff
changeset
|
50 |
local refresh_interval = module:get_option_number("feed_pull_interval_seconds", legacy_refresh_interval*60); |
2405
7a1625a84624
mod_pubsub_feeds: Ask for leases that expire after one day (Years after testing this module, I was still subscribed to a GNU Social instance)
Kim Alvefur <zash@zash.se>
parents:
2388
diff
changeset
|
51 |
local lease_length = tostring(math.floor(module:get_option_number("feed_lease_length", 86400))); |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
52 |
|
794
5a8fa70faa13
mod_pubsub_feeds: Remove config-reloaded hook, do setup in module.load() instead
Kim Alvefur <zash@zash.se>
parents:
793
diff
changeset
|
53 |
function module.load() |
2381
02532f28b207
mod_pubsub_feeds: Remove default feeds
Kim Alvefur <zash@zash.se>
parents:
2141
diff
changeset
|
54 |
local config = module:get_option("feeds", { }); |
797
68a067d5ed3b
mod_pubsub_feeds: util.pubsub methods return bool, data
Kim Alvefur <zash@zash.se>
parents:
796
diff
changeset
|
55 |
local ok, nodes = pubsub.service:get_nodes(true); |
68a067d5ed3b
mod_pubsub_feeds: util.pubsub methods return bool, data
Kim Alvefur <zash@zash.se>
parents:
796
diff
changeset
|
56 |
if not ok then nodes = {}; end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
57 |
local new_feed_list = {}; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
58 |
for node, url in pairs(config) do |
765
82d292e4f30f
mod_pubsub_feeds: Use URL as node name if the config index is numeric
Kim Alvefur <zash@zash.se>
parents:
763
diff
changeset
|
59 |
if type(node) == "number" then |
82d292e4f30f
mod_pubsub_feeds: Use URL as node name if the config index is numeric
Kim Alvefur <zash@zash.se>
parents:
763
diff
changeset
|
60 |
node = url; |
82d292e4f30f
mod_pubsub_feeds: Use URL as node name if the config index is numeric
Kim Alvefur <zash@zash.se>
parents:
763
diff
changeset
|
61 |
end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
62 |
new_feed_list[node] = true; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
63 |
if not feed_list[node] then |
5576
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
64 |
local ok, err = pubsub.service:create(node, true); |
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
65 |
if ok then |
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
66 |
feed_list[node] = { url = url; node = node; last_update = 0 }; |
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
67 |
else |
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
68 |
module:log("error", "Could not create node %s: %s", node, err); |
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
69 |
end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
70 |
else |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
71 |
feed_list[node].url = url; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
72 |
end |
795
c7f507a190a0
mod_pubsub_feeds: Check if nodes exists in pubsub, make sure they get restored if not. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
794
diff
changeset
|
73 |
if not nodes[node] then |
c7f507a190a0
mod_pubsub_feeds: Check if nodes exists in pubsub, make sure they get restored if not. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
794
diff
changeset
|
74 |
feed_list[node].last_update = 0; |
c7f507a190a0
mod_pubsub_feeds: Check if nodes exists in pubsub, make sure they get restored if not. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
794
diff
changeset
|
75 |
end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
76 |
end |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
77 |
for node in pairs(feed_list) do |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
78 |
if not new_feed_list[node] then |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
79 |
feed_list[node] = nil; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
80 |
end |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
81 |
end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
82 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
83 |
|
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
84 |
function update_entry(item) |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
85 |
local node = item.node; |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
86 |
module:log("debug", "parsing %d bytes of data in node %s", #item.data or 0, node) |
3285
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3284
diff
changeset
|
87 |
local feed, err = parse_feed(item.data); |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3284
diff
changeset
|
88 |
if not feed then |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3284
diff
changeset
|
89 |
module:log("error", "Could not parse feed %q: %s", item.url, err); |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3284
diff
changeset
|
90 |
module:log("debug", "Feed data:\n%s\n.", item.data); |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3284
diff
changeset
|
91 |
return; |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3284
diff
changeset
|
92 |
end |
2383
bc6e63ab2a7c
mod_pubsub_feeds: Iterate over feed backwards due to most feeds being in reverse chronological order
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
93 |
local entries = {}; |
2136
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
94 |
for entry in feed:childtags("entry") do |
2383
bc6e63ab2a7c
mod_pubsub_feeds: Iterate over feed backwards due to most feeds being in reverse chronological order
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
95 |
table.insert(entries, entry); |
bc6e63ab2a7c
mod_pubsub_feeds: Iterate over feed backwards due to most feeds being in reverse chronological order
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
96 |
end |
2387
18323c6aa133
mod_pubsub_feeds: Halt feed parsing when we see a node we already know about
Kim Alvefur <zash@zash.se>
parents:
2386
diff
changeset
|
97 |
local ok, items = pubsub.service:get_items(node, true); |
2386
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2385
diff
changeset
|
98 |
if not ok then |
5576
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
99 |
module:log("error", "PubSub node %q missing: %s", node, items); |
fd1c535dcb92
mod_pubsub_feeds: Create pubsub nodes on module load instead of later
Kim Alvefur <zash@zash.se>
parents:
5575
diff
changeset
|
100 |
return |
2386
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2385
diff
changeset
|
101 |
end |
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
102 |
|
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
103 |
local start_from = #entries; |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
104 |
for i, entry in ipairs(entries) do |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
105 |
local id = entry:get_child_text("id"); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
106 |
if not id then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
107 |
local link = entry:get_child("link"); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
108 |
if link then |
5577
ed177fa5b54c
mod_pubsub_feeds: Fix mixup between feed object and parsed feed
Kim Alvefur <zash@zash.se>
parents:
5576
diff
changeset
|
109 |
module:log("debug", "Feed %q item %s is missing an id, using <link> instead", item.url, entry:top_tag()); |
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
110 |
id = link and link.attr.href; |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
111 |
else |
5578
10b7118bbd8f
mod_pubsub_feeds: Remove broken attempt to generate an ID from content
Kim Alvefur <zash@zash.se>
parents:
5577
diff
changeset
|
112 |
module:log("error", "Feed %q item %s is missing both id and link, this feed is unusable", item.url, entry:top_tag()); |
10b7118bbd8f
mod_pubsub_feeds: Remove broken attempt to generate an ID from content
Kim Alvefur <zash@zash.se>
parents:
5577
diff
changeset
|
113 |
return; |
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
114 |
end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
115 |
entry:text_tag("id", id); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
116 |
end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
117 |
|
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
118 |
if items[id] then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
119 |
-- This should be the first item that we already have. |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
120 |
start_from = i-1; |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
121 |
break |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
122 |
end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
123 |
end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
124 |
|
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
125 |
for i = start_from, 1, -1 do -- Feeds are usually in reverse order |
2385
a4614a329bdd
mod_pubsub_feeds: Get entry from list of entries (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents:
2384
diff
changeset
|
126 |
local entry = entries[i]; |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
127 |
entry.attr.xmlns = xmlns_atom; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
128 |
|
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
129 |
local id = entry:get_child_text("id"); |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
130 |
|
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
131 |
local timestamp = dt_parse(entry:get_child_text("published")); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
132 |
if not timestamp then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
133 |
timestamp = time(); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
134 |
entry:text_tag("published", dt_datetime(timestamp)); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
135 |
end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
136 |
|
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
137 |
if not timestamp or not item.last_update or timestamp > item.last_update then |
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
138 |
local xitem = st.stanza("item", { id = id, xmlns = "http://jabber.org/protocol/pubsub" }):add_child(entry); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
139 |
-- TODO Put data from /feed into item/source |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
140 |
|
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
141 |
local ok, err = pubsub.service:publish(node, true, id, xitem); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
142 |
if not ok then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
143 |
module:log("error", "Publishing to node %s failed: %s", node, err); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
144 |
elseif timestamp then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
145 |
item.last_update = timestamp; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
146 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
147 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
148 |
end |
1343
7dbde05b48a9
all the things: Remove trailing whitespace
Florian Zeitz <florob@babelmonkeys.de>
parents:
1325
diff
changeset
|
149 |
|
1457
720ff25d94e6
mod_pubsub_feeds: Check that lease time exists before comparing
Kim Alvefur <zash@zash.se>
parents:
1456
diff
changeset
|
150 |
if item.lease_expires and item.lease_expires > time() then |
1455
13e359c48b5b
mod_pubsub_feeds: Fix refreshing of subscriptions
Kim Alvefur <zash@zash.se>
parents:
1454
diff
changeset
|
151 |
item.subscription = nil; |
13e359c48b5b
mod_pubsub_feeds: Fix refreshing of subscriptions
Kim Alvefur <zash@zash.se>
parents:
1454
diff
changeset
|
152 |
item.lease_expires = nil; |
13e359c48b5b
mod_pubsub_feeds: Fix refreshing of subscriptions
Kim Alvefur <zash@zash.se>
parents:
1454
diff
changeset
|
153 |
end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
154 |
if use_pubsubhubub and not item.subscription then |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
155 |
--module:log("debug", "check if %s has a hub", item.node); |
2136
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
156 |
for link in feed:childtags("link") do |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
157 |
if link.attr.rel == "hub" then |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
158 |
item.hub = link.attr.href; |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
159 |
module:log("debug", "Node %s has a hub: %s", item.node, item.hub); |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
160 |
return subscribe(item); |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
161 |
end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
162 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
163 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
164 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
165 |
|
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
166 |
function fetch(item, callback) -- HTTP Pull |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
167 |
local headers = { }; |
3051
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
168 |
if item.data and item.etag then |
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
169 |
headers["If-None-Match"] = item.etag; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
170 |
end |
3051
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
171 |
http.request(item.url, { headers = headers }, function(data, code, resp) |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
172 |
if code == 200 then |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
173 |
item.data = data; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
174 |
if callback then callback(item) end |
3051
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
175 |
if resp.headers then |
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
176 |
item.etag = resp.headers.etag |
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2649
diff
changeset
|
177 |
end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
178 |
elseif code == 304 then |
5575
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5574
diff
changeset
|
179 |
module:log("debug", "No updates to %q", item.url); |
3283
e6b25054c110
mod_pubsub_feeds: Log a message for HTTP 301 redirects
Kim Alvefur <zash@zash.se>
parents:
3282
diff
changeset
|
180 |
elseif code == 301 and resp.headers.location then |
e6b25054c110
mod_pubsub_feeds: Log a message for HTTP 301 redirects
Kim Alvefur <zash@zash.se>
parents:
3282
diff
changeset
|
181 |
module:log("info", "Feed %q has moved to %q", item.url, resp.headers.location); |
3284
645c4f14e03d
mod_pubsub_feeds: Fix syntax error
Kim Alvefur <zash@zash.se>
parents:
3283
diff
changeset
|
182 |
elseif code <= 100 then |
3282
958f0b8b502c
mod_pubsub_feeds: Log non-HTTP errors like certificate problems etc
Kim Alvefur <zash@zash.se>
parents:
3281
diff
changeset
|
183 |
module:log("error", "Error fetching %q: %q[%d]", item.url, data, code); |
3281
78b11fb291a2
mod_pubsub_feeds: Log a debug message for status codes not handled
Kim Alvefur <zash@zash.se>
parents:
3246
diff
changeset
|
184 |
else |
78b11fb291a2
mod_pubsub_feeds: Log a debug message for status codes not handled
Kim Alvefur <zash@zash.se>
parents:
3246
diff
changeset
|
185 |
module:log("debug", "Unhandled status code %d when fetching %q", code, item.url); |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
186 |
end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
187 |
end); |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
188 |
end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
189 |
|
1453
7b53cfc6ba8d
mod_pubsub_feeds: Fix refresh_feeds() allways refreshing all feeds
Kim Alvefur <zash@zash.se>
parents:
1343
diff
changeset
|
190 |
function refresh_feeds(now) |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
191 |
--module:log("debug", "Refreshing feeds"); |
2138
bd7744df0b4a
mod_pubsub_feeds: Remove unused imports [luacheck]
Kim Alvefur <zash@zash.se>
parents:
2136
diff
changeset
|
192 |
for _, item in pairs(feed_list) do |
1453
7b53cfc6ba8d
mod_pubsub_feeds: Fix refresh_feeds() allways refreshing all feeds
Kim Alvefur <zash@zash.se>
parents:
1343
diff
changeset
|
193 |
if item.subscription ~= "subscribe" and item.last_update + refresh_interval < now then |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
194 |
--module:log("debug", "checking %s", item.node); |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
195 |
fetch(item, update_entry); |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
196 |
end |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
197 |
end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
198 |
return refresh_interval; |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
199 |
end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
200 |
|
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
201 |
local function format_url(node) |
2138
bd7744df0b4a
mod_pubsub_feeds: Remove unused imports [luacheck]
Kim Alvefur <zash@zash.se>
parents:
2136
diff
changeset
|
202 |
return module:http_url(nil, "/callback") .. "?" .. formencode({ node = node }); |
1343
7dbde05b48a9
all the things: Remove trailing whitespace
Florian Zeitz <florob@babelmonkeys.de>
parents:
1325
diff
changeset
|
203 |
end |
401
c85397063eca
mod_pubsub_feed: Try harder to figure out ones callback URL.
Kim Alvefur <zash@zash.se>
parents:
400
diff
changeset
|
204 |
|
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
205 |
function subscribe(feed, want) |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
206 |
want = want or "subscribe"; |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
207 |
feed.secret = feed.secret or uuid(); |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
208 |
local body = formencode{ |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
209 |
["hub.callback"] = format_url(feed.node); |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
210 |
["hub.mode"] = want; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
211 |
["hub.topic"] = feed.url; |
1454
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
212 |
["hub.verify"] = "async"; -- COMPAT this is REQUIRED in the 0.3 draft but removed in 0.4 |
325
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
213 |
["hub.secret"] = feed.secret; |
2405
7a1625a84624
mod_pubsub_feeds: Ask for leases that expire after one day (Years after testing this module, I was still subscribed to a GNU Social instance)
Kim Alvefur <zash@zash.se>
parents:
2388
diff
changeset
|
214 |
["hub.lease_seconds"] = lease_length; |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
215 |
}; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
216 |
|
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
217 |
--module:log("debug", "subscription request, body: %s", body); |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
218 |
|
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
219 |
--FIXME The subscription states and related stuff |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
220 |
feed.subscription = want; |
983
ac9bf3fcbcfe
mod_pubsub_feeds, mod_sms_clickatell, mod_twitter: Update for net.http API change in prosody:e3b9dc9dd940
Matthew Wild <mwild1@gmail.com>
parents:
797
diff
changeset
|
221 |
http.request(feed.hub, { body = body }, function(data, code) |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
222 |
module:log("debug", "subscription to %s submitted, status %s", feed.node, tostring(code)); |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
223 |
if code >= 400 then |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
224 |
module:log("error", "There was something wrong with our subscription request, body: %s", tostring(data)); |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
225 |
feed.subscription = "failed"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
226 |
end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
227 |
end); |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
228 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
229 |
|
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
230 |
function handle_http_request(event) |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
231 |
local request = event.request; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
232 |
local method = request.method; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
233 |
local body = request.body; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
234 |
|
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
235 |
local query = request.url.query or {}; --FIXME |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
236 |
if query and type(query) == "string" then |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
237 |
query = formdecode(query); |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
238 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
239 |
|
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
240 |
local feed = feed_list[query.node]; |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
241 |
if not feed then |
2426
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2425
diff
changeset
|
242 |
if query["hub.mode"] == "unsubscribe" then |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2425
diff
changeset
|
243 |
-- Unsubscribe from unknown feed |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2425
diff
changeset
|
244 |
module:log("debug", "Unsubscribe from unknown feed %s -- %s", query["hub.topic"], formencode(query)); |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2425
diff
changeset
|
245 |
return query["hub.challenge"]; |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2425
diff
changeset
|
246 |
end |
2425
a9add2d1adf5
mod_pubsub_feeds: Log pushes for unknown feeds to aid in manual unsubscription from them
Kim Alvefur <zash@zash.se>
parents:
2405
diff
changeset
|
247 |
module:log("debug", "Push for unknown feed %s -- %s", query["hub.topic"], formencode(query)); |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
248 |
return 404; |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
249 |
end |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
250 |
|
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
251 |
if method == "GET" then |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
252 |
if query.node then |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
253 |
if query["hub.topic"] ~= feed.url then |
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
254 |
module:log("debug", "Invalid topic: %s", tostring(query["hub.topic"])) |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
255 |
return 404 |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
256 |
end |
1456
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
257 |
if query["hub.mode"] == "denied" then |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
258 |
module:log("info", "Subscription denied: %s", tostring(query["hub.reason"] or "No reason given")) |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
259 |
feed.subscription = "denied"; |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
260 |
return "Ok then :("; |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
261 |
elseif query["hub.mode"] == feed.subscription then |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
262 |
module:log("debug", "Confirming %s request to %s", feed.subscription, feed.url) |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
263 |
else |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
264 |
module:log("debug", "Invalid mode: %s", tostring(query["hub.mode"])) |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
265 |
return 400 |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
266 |
end |
1454
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
267 |
local lease_seconds = tonumber(query["hub.lease_seconds"]); |
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
268 |
if lease_seconds then |
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
269 |
feed.lease_expires = time() + lease_seconds - refresh_interval * 2; |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
270 |
end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
271 |
return query["hub.challenge"]; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
272 |
end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
273 |
return 400; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
274 |
elseif method == "POST" then |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
275 |
if #body > 0 then |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
276 |
module:log("debug", "got %d bytes PuSHed for %s", #body, query.node); |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
277 |
local signature = request.headers.x_hub_signature; |
325
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
278 |
if feed.secret then |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
279 |
local localsig = "sha1=" .. hmac_sha1(feed.secret, body, true); |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
280 |
if localsig ~= signature then |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
281 |
module:log("debug", "Invalid signature, got %s but wanted %s", tostring(signature), tostring(localsig)); |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
282 |
return 401; |
325
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
283 |
end |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
284 |
module:log("debug", "Valid signature"); |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
285 |
end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
286 |
feed.data = body; |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
287 |
update_entry(feed); |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
288 |
return 202; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
289 |
end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
290 |
return 400; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
291 |
end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
292 |
return 501; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
293 |
end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
294 |
|
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
295 |
if use_pubsubhubub then |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
296 |
module:provides("http", { |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
297 |
default_path = "/callback"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
298 |
route = { |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
299 |
GET = handle_http_request; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
300 |
POST = handle_http_request; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
301 |
-- This all? |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
302 |
}; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
303 |
}); |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
304 |
end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
305 |
|
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
306 |
module:add_timer(1, refresh_feeds); |