From 8d1a26ba72816354030a339f819dff7282068db4 Mon Sep 17 00:00:00 2001
From: Torsten Grote <t@grobox.de>
Date: Mon, 25 Jul 2016 15:46:26 -0300
Subject: [PATCH] Keep track of which RSS entries have been seen
This is done by remembering the time of the latest entry.
All entries newer than that are considered new and will be posted.
Closes #485
---
.../src/org/briarproject/api/feed/Feed.java | 24 +++++++--
.../briarproject/api/feed/FeedConstants.java | 1 +
.../briarproject/feed/FeedManagerImpl.java | 53 ++++++++++++++++---
3 files changed, 68 insertions(+), 10 deletions(-)
diff --git a/briar-api/src/org/briarproject/api/feed/Feed.java b/briar-api/src/org/briarproject/api/feed/Feed.java
index 0f6af8a2d1..7b39486698 100644
--- a/briar-api/src/org/briarproject/api/feed/Feed.java
+++ b/briar-api/src/org/briarproject/api/feed/Feed.java
@@ -10,6 +10,7 @@ import static org.briarproject.api.feed.FeedConstants.KEY_BLOG_GROUP_ID;
import static org.briarproject.api.feed.FeedConstants.KEY_FEED_ADDED;
import static org.briarproject.api.feed.FeedConstants.KEY_FEED_AUTHOR;
import static org.briarproject.api.feed.FeedConstants.KEY_FEED_DESC;
+import static org.briarproject.api.feed.FeedConstants.KEY_FEED_LAST_ENTRY;
import static org.briarproject.api.feed.FeedConstants.KEY_FEED_TITLE;
import static org.briarproject.api.feed.FeedConstants.KEY_FEED_UPDATED;
import static org.briarproject.api.feed.FeedConstants.KEY_FEED_URL;
@@ -19,11 +20,11 @@ public class Feed {
final private String url;
final private GroupId blogId;
final private String title, description, author;
- final private long added, updated;
+ final private long added, updated, lastEntryTime;
public Feed(String url, GroupId blogId, @Nullable String title,
@Nullable String description, @Nullable String author,
- long added, long updated) {
+ long added, long updated, long lastEntryTime) {
this.url = url;
this.blogId = blogId;
@@ -32,6 +33,14 @@ public class Feed {
this.author = author;
this.added = added;
this.updated = updated;
+ this.lastEntryTime = lastEntryTime;
+ }
+
+ public Feed(String url, GroupId blogId, @Nullable String title,
+ @Nullable String description, @Nullable String author,
+ long added) {
+
+ this(url, blogId, title, description, author, added, 0L, 0L);
}
public String getUrl() {
@@ -47,7 +56,8 @@ public class Feed {
new BdfEntry(KEY_FEED_URL, url),
new BdfEntry(KEY_BLOG_GROUP_ID, blogId.getBytes()),
new BdfEntry(KEY_FEED_ADDED, added),
- new BdfEntry(KEY_FEED_UPDATED, updated)
+ new BdfEntry(KEY_FEED_UPDATED, updated),
+ new BdfEntry(KEY_FEED_LAST_ENTRY, lastEntryTime)
);
if (title != null) d.put(KEY_FEED_TITLE, title);
if (description != null) d.put(KEY_FEED_DESC, description);
@@ -63,7 +73,9 @@ public class Feed {
String author = d.getOptionalString(KEY_FEED_AUTHOR);
long added = d.getLong(KEY_FEED_ADDED, 0L);
long updated = d.getLong(KEY_FEED_UPDATED, 0L);
- return new Feed(url, blogId, title, desc, author, added, updated);
+ long lastEntryTime = d.getLong(KEY_FEED_LAST_ENTRY, 0L);
+ return new Feed(url, blogId, title, desc, author, added, updated,
+ lastEntryTime);
}
public String getTitle() {
@@ -86,4 +98,8 @@ public class Feed {
return updated;
}
+ public long getLastEntryTime() {
+ return lastEntryTime;
+ }
+
}
diff --git a/briar-api/src/org/briarproject/api/feed/FeedConstants.java b/briar-api/src/org/briarproject/api/feed/FeedConstants.java
index 5a38c979f6..68117a2f44 100644
--- a/briar-api/src/org/briarproject/api/feed/FeedConstants.java
+++ b/briar-api/src/org/briarproject/api/feed/FeedConstants.java
@@ -17,5 +17,6 @@ public interface FeedConstants {
String KEY_FEED_AUTHOR = "feedAuthor";
String KEY_FEED_ADDED = "feedAdded";
String KEY_FEED_UPDATED = "feedUpdated";
+ String KEY_FEED_LAST_ENTRY = "feedLastEntryTime";
}
diff --git a/briar-core/src/org/briarproject/feed/FeedManagerImpl.java b/briar-core/src/org/briarproject/feed/FeedManagerImpl.java
index 40a7a5a2f2..f0ea3eed6c 100644
--- a/briar-core/src/org/briarproject/feed/FeedManagerImpl.java
+++ b/briar-core/src/org/briarproject/feed/FeedManagerImpl.java
@@ -1,5 +1,7 @@
package org.briarproject.feed;
+import com.rometools.rome.feed.synd.SyndContent;
+import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
@@ -121,14 +123,17 @@ class FeedManagerImpl implements FeedManager, Service, Client {
@Override
public void addFeed(String url, GroupId g) throws DbException, IOException {
+ LOG.info("Adding new RSS feed...");
Feed feed;
+ // TODO check for existing feed?
try {
SyndFeed f = getSyndFeed(getFeedInputStream(url));
- String title = f.getTitle();
+ String title = StringUtils.isNullOrEmpty(f.getTitle()) ? null :
+ f.getTitle();
String description = f.getDescription();
String author = f.getAuthor();
long added = System.currentTimeMillis();
- feed = new Feed(url, g, title, description, author, added, added);
+ feed = new Feed(url, g, title, description, author, added);
} catch (FeedException e) {
throw new IOException(e);
}
@@ -153,6 +158,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
if (feed.getUrl().equals(url)) {
found = true;
feeds.remove(feed);
+ break;
}
}
if (!found) throw new DbException();
@@ -217,6 +223,8 @@ class FeedManagerImpl implements FeedManager, Service, Client {
}
private void fetchFeeds() {
+ LOG.info("Updating RSS feeds...");
+
// Get current feeds
List<Feed> feeds;
try {
@@ -243,17 +251,50 @@ class FeedManagerImpl implements FeedManager, Service, Client {
}
private Feed fetchFeed(Feed feed) {
- LOG.info("Updating RSS feeds...");
String title, description, author;
long updated = System.currentTimeMillis();
+ long lastEntryTime = feed.getLastEntryTime();
try {
SyndFeed f = getSyndFeed(getFeedInputStream(feed.getUrl()));
title = f.getTitle();
description = f.getDescription();
author = f.getAuthor();
- // TODO keep track of which entries have been seen (#485)
- // TODO Pass any new entries down the pipeline to be posted (#486)
+ LOG.info("Title: " + f.getTitle());
+ LOG.info("Description: " + f.getDescription());
+ LOG.info("Author: " + f.getAuthor());
+ LOG.info("Number of Entries: " + f.getEntries().size());
+ LOG.info("------------------------------");
+
+ for (SyndEntry entry : f.getEntries()) {
+ LOG.info("Entry Title: " + entry.getTitle());
+ LOG.info("Entry Author: " + entry.getAuthor());
+ LOG.info("Entry Published Date: " + entry.getPublishedDate());
+ LOG.info("Entry Updated Date: " + entry.getUpdatedDate());
+ LOG.info("Entry Link: " + entry.getLink());
+ LOG.info("Entry URI: " + entry.getUri());
+ //LOG.info("Entry Description: " + entry.getDescription());
+ long entryTime;
+ if (entry.getPublishedDate() != null) {
+ entryTime = entry.getPublishedDate().getTime();
+ } else if (entry.getUpdatedDate() != null) {
+ entryTime = entry.getUpdatedDate().getTime();
+ } else {
+ // no time information available, ignore this entry
+ if (LOG.isLoggable(WARNING))
+ LOG.warning("Entry has no date: " + entry.getTitle());
+ continue;
+ }
+ if (entryTime > feed.getLastEntryTime()) {
+ LOG.info("Adding new entry...");
+ // TODO Pass any new entries down the pipeline to be posted (#486)
+ for (SyndContent content : entry.getContents()) {
+ LOG.info("Content: " + content.getValue());
+ }
+ if (entryTime > lastEntryTime) lastEntryTime = entryTime;
+ }
+ LOG.info("------------------------------");
+ }
} catch (FeedException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
@@ -264,7 +305,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
return feed;
}
return new Feed(feed.getUrl(), feed.getBlogId(), title, description,
- author, feed.getAdded(), updated);
+ author, feed.getAdded(), updated, lastEntryTime);
}
private InputStream getFeedInputStream(String url) throws IOException {
--
GitLab