diff --git a/briar-api/src/org/briarproject/api/feed/Feed.java b/briar-api/src/org/briarproject/api/feed/Feed.java
index 7b394866989253bddaf2412359ce96330a4a489a..592e0d5432584835a229392e19704cf6dbea9693 100644
--- a/briar-api/src/org/briarproject/api/feed/Feed.java
+++ b/briar-api/src/org/briarproject/api/feed/Feed.java
@@ -43,6 +43,11 @@ public class Feed {
this(url, blogId, title, description, author, added, 0L, 0L);
}
+ public Feed(String url, GroupId blogId, long added) {
+
+ this(url, blogId, null, null, null, added, 0L, 0L);
+ }
+
public String getUrl() {
return url;
}
diff --git a/briar-api/src/org/briarproject/api/feed/FeedConstants.java b/briar-api/src/org/briarproject/api/feed/FeedConstants.java
index 68117a2f44bd31bd3bfff6e81341d54551161e1b..bc98ad7eeace3ffa0274608161925786e7117d5c 100644
--- a/briar-api/src/org/briarproject/api/feed/FeedConstants.java
+++ b/briar-api/src/org/briarproject/api/feed/FeedConstants.java
@@ -1,13 +1,20 @@
package org.briarproject.api.feed;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+
public interface FeedConstants {
- /* delay after start before fetching feed, in minutes */
+ /* delay after start before fetching feed */
int FETCH_DELAY_INITIAL = 1;
- /* the interval the feed should be fetched, in minutes */
+ /* the interval the feed should be fetched */
int FETCH_INTERVAL = 30;
+ /* the unit that applies to the fetch times */
+ TimeUnit FETCH_UNIT = MINUTES;
+
// group metadata keys
String KEY_FEEDS = "feeds";
String KEY_FEED_URL = "feedURL";
diff --git a/briar-core/src/org/briarproject/blogs/BlogManagerImpl.java b/briar-core/src/org/briarproject/blogs/BlogManagerImpl.java
index 182ea11869bb779e697910b5c9c73f5230ad75e9..a35bc0bc7fbad33d9b2928941fec1e91668ab16a 100644
--- a/briar-core/src/org/briarproject/blogs/BlogManagerImpl.java
+++ b/briar-core/src/org/briarproject/blogs/BlogManagerImpl.java
@@ -191,6 +191,7 @@ class BlogManagerImpl extends BdfIncomingMessageHook implements BlogManager,
@Override
public void removeBlog(Blog b) throws DbException {
+ // TODO if this gets used, check for RSS feeds posting into this blog
Transaction txn = db.startTransaction(false);
try {
for (RemoveBlogHook hook : removeHooks)
diff --git a/briar-core/src/org/briarproject/feed/FeedManagerImpl.java b/briar-core/src/org/briarproject/feed/FeedManagerImpl.java
index f0ea3eed6c24e1b419448c1aa5446ff34da88d6d..cf36c7be7e5ef91d32d9cd7dcd93d697823dedee 100644
--- a/briar-core/src/org/briarproject/feed/FeedManagerImpl.java
+++ b/briar-core/src/org/briarproject/feed/FeedManagerImpl.java
@@ -8,7 +8,10 @@ import com.rometools.rome.io.SyndFeedInput;
import com.rometools.rome.io.XmlReader;
import org.briarproject.api.FormatException;
+import org.briarproject.api.blogs.Blog;
import org.briarproject.api.blogs.BlogManager;
+import org.briarproject.api.blogs.BlogPost;
+import org.briarproject.api.blogs.BlogPostFactory;
import org.briarproject.api.clients.Client;
import org.briarproject.api.clients.ClientHelper;
import org.briarproject.api.clients.PrivateGroupFactory;
@@ -21,18 +24,27 @@ import org.briarproject.api.db.Transaction;
import org.briarproject.api.feed.Feed;
import org.briarproject.api.feed.FeedManager;
import org.briarproject.api.lifecycle.IoExecutor;
+import org.briarproject.api.identity.AuthorId;
+import org.briarproject.api.identity.IdentityManager;
+import org.briarproject.api.identity.LocalAuthor;
import org.briarproject.api.lifecycle.Service;
import org.briarproject.api.lifecycle.ServiceException;
import org.briarproject.api.sync.ClientId;
import org.briarproject.api.sync.Group;
import org.briarproject.api.sync.GroupId;
+import org.briarproject.api.system.Clock;
import org.briarproject.util.StringUtils;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.Proxy;
+import java.security.GeneralSecurityException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
@@ -44,10 +56,12 @@ import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
-import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.logging.Level.INFO;
import static java.util.logging.Level.WARNING;
+import static org.briarproject.api.blogs.BlogConstants.MAX_BLOG_POST_BODY_LENGTH;
import static org.briarproject.api.feed.FeedConstants.FETCH_DELAY_INITIAL;
import static org.briarproject.api.feed.FeedConstants.FETCH_INTERVAL;
+import static org.briarproject.api.feed.FeedConstants.FETCH_UNIT;
import static org.briarproject.api.feed.FeedConstants.KEY_FEEDS;
class FeedManagerImpl implements FeedManager, Service, Client {
@@ -65,19 +79,28 @@ class FeedManagerImpl implements FeedManager, Service, Client {
private final DatabaseComponent db;
private final PrivateGroupFactory privateGroupFactory;
private final ClientHelper clientHelper;
+ private IdentityManager identityManager;
private final BlogManager blogManager;
+ @Inject
+ @SuppressWarnings("WeakerAccess")
+ BlogPostFactory blogPostFactory;
+ @Inject
+ @SuppressWarnings("WeakerAccess")
+ Clock clock;
+
@Inject
FeedManagerImpl(ScheduledExecutorService feedExecutor,
@IoExecutor Executor ioExecutor, DatabaseComponent db,
PrivateGroupFactory privateGroupFactory, ClientHelper clientHelper,
- BlogManager blogManager) {
+ IdentityManager identityManager, BlogManager blogManager) {
this.feedExecutor = feedExecutor;
this.ioExecutor = ioExecutor;
this.db = db;
this.privateGroupFactory = privateGroupFactory;
this.clientHelper = clientHelper;
+ this.identityManager = identityManager;
this.blogManager = blogManager;
}
@@ -99,7 +122,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
}
};
feedExecutor.scheduleWithFixedDelay(fetcher, FETCH_DELAY_INITIAL,
- FETCH_INTERVAL, MINUTES);
+ FETCH_INTERVAL, FETCH_UNIT);
}
@Override
@@ -124,16 +147,11 @@ class FeedManagerImpl implements FeedManager, Service, Client {
@Override
public void addFeed(String url, GroupId g) throws DbException, IOException {
LOG.info("Adding new RSS feed...");
- Feed feed;
+
// TODO check for existing feed?
+ Feed feed = new Feed(url, g, clock.currentTimeMillis());
try {
- SyndFeed f = getSyndFeed(getFeedInputStream(url));
- String title = StringUtils.isNullOrEmpty(f.getTitle()) ? null :
- f.getTitle();
- String description = f.getDescription();
- String author = f.getAuthor();
- long added = System.currentTimeMillis();
- feed = new Feed(url, g, title, description, author, added);
+ feed = fetchFeed(feed);
} catch (FeedException e) {
throw new IOException(e);
}
@@ -142,7 +160,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
try {
List<Feed> feeds = getFeeds(txn);
feeds.add(feed);
- storeFeeds(feeds);
+ storeFeeds(txn, feeds);
} finally {
db.endTransaction(txn);
}
@@ -150,6 +168,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
@Override
public void removeFeed(String url) throws DbException {
+ LOG.info("Removing RSS feed...");
Transaction txn = db.startTransaction(false);
try {
List<Feed> feeds = getFeeds(txn);
@@ -222,6 +241,11 @@ class FeedManagerImpl implements FeedManager, Service, Client {
storeFeeds(null, feeds);
}
+ /**
+ * This method is called periodically from a background service.
+ * It fetches all available feeds and posts new entries to the respective
+ * blog.
+ */
private void fetchFeeds() {
LOG.info("Updating RSS feeds...");
@@ -238,7 +262,15 @@ class FeedManagerImpl implements FeedManager, Service, Client {
// Fetch and update all feeds
List<Feed> newFeeds = new ArrayList<Feed>(feeds.size());
for (Feed feed : feeds) {
- newFeeds.add(fetchFeed(feed));
+ try {
+ newFeeds.add(fetchFeed(feed));
+ } catch (FeedException e) {
+ if (LOG.isLoggable(WARNING))
+ LOG.log(WARNING, e.toString(), e);
+ } catch (IOException e) {
+ if (LOG.isLoggable(WARNING))
+ LOG.log(WARNING, e.toString(), e);
+ }
}
// Store updated feeds
@@ -248,61 +280,45 @@ class FeedManagerImpl implements FeedManager, Service, Client {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
}
+ LOG.info("Done updating RSS feeds");
}
- private Feed fetchFeed(Feed feed) {
+ private Feed fetchFeed(Feed feed) throws FeedException, IOException {
+ if (LOG.isLoggable(INFO))
+ LOG.info("Fetching feed from " + feed.getUrl());
+
String title, description, author;
- long updated = System.currentTimeMillis();
+ long updated = clock.currentTimeMillis();
long lastEntryTime = feed.getLastEntryTime();
- try {
- SyndFeed f = getSyndFeed(getFeedInputStream(feed.getUrl()));
- title = f.getTitle();
- description = f.getDescription();
- author = f.getAuthor();
-
- LOG.info("Title: " + f.getTitle());
- LOG.info("Description: " + f.getDescription());
- LOG.info("Author: " + f.getAuthor());
- LOG.info("Number of Entries: " + f.getEntries().size());
- LOG.info("------------------------------");
-
- for (SyndEntry entry : f.getEntries()) {
- LOG.info("Entry Title: " + entry.getTitle());
- LOG.info("Entry Author: " + entry.getAuthor());
- LOG.info("Entry Published Date: " + entry.getPublishedDate());
- LOG.info("Entry Updated Date: " + entry.getUpdatedDate());
- LOG.info("Entry Link: " + entry.getLink());
- LOG.info("Entry URI: " + entry.getUri());
- //LOG.info("Entry Description: " + entry.getDescription());
- long entryTime;
- if (entry.getPublishedDate() != null) {
- entryTime = entry.getPublishedDate().getTime();
- } else if (entry.getUpdatedDate() != null) {
- entryTime = entry.getUpdatedDate().getTime();
- } else {
- // no time information available, ignore this entry
- if (LOG.isLoggable(WARNING))
- LOG.warning("Entry has no date: " + entry.getTitle());
- continue;
- }
- if (entryTime > feed.getLastEntryTime()) {
- LOG.info("Adding new entry...");
- // TODO Pass any new entries down the pipeline to be posted (#486)
- for (SyndContent content : entry.getContents()) {
- LOG.info("Content: " + content.getValue());
- }
- if (entryTime > lastEntryTime) lastEntryTime = entryTime;
- }
- LOG.info("------------------------------");
+
+ SyndFeed f = getSyndFeed(getFeedInputStream(feed.getUrl()));
+ title = StringUtils.isNullOrEmpty(f.getTitle()) ? null : f.getTitle();
+ if (title != null) title = stripHTML(title);
+ description = StringUtils.isNullOrEmpty(f.getDescription()) ? null :
+ f.getDescription();
+ if (description != null) description = stripHTML(description);
+ author =
+ StringUtils.isNullOrEmpty(f.getAuthor()) ? null : f.getAuthor();
+ if (author != null) author = stripHTML(author);
+
+ // sort and add new entries
+ Collections.sort(f.getEntries(), getEntryComparator());
+ for (SyndEntry entry : f.getEntries()) {
+ long entryTime;
+ if (entry.getPublishedDate() != null) {
+ entryTime = entry.getPublishedDate().getTime();
+ } else if (entry.getUpdatedDate() != null) {
+ entryTime = entry.getUpdatedDate().getTime();
+ } else {
+ // no time information available, ignore this entry
+ if (LOG.isLoggable(WARNING))
+ LOG.warning("Entry has no date: " + entry.getTitle());
+ continue;
+ }
+ if (entryTime > feed.getLastEntryTime()) {
+ postEntry(feed, entry);
+ if (entryTime > lastEntryTime) lastEntryTime = entryTime;
}
- } catch (FeedException e) {
- if (LOG.isLoggable(WARNING))
- LOG.log(WARNING, e.toString(), e);
- return feed;
- } catch (IOException e) {
- if (LOG.isLoggable(WARNING))
- LOG.log(WARNING, e.toString(), e);
- return feed;
}
return new Feed(feed.getUrl(), feed.getBlogId(), title, description,
author, feed.getAdded(), updated, lastEntryTime);
@@ -313,7 +329,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
// TODO verify and use local Tor proxy address/port
String proxyHost = "localhost";
int proxyPort = 59050;
- Proxy proxy = new Proxy(Proxy.Type.HTTP,
+ Proxy proxy = new Proxy(Proxy.Type.SOCKS,
new InetSocketAddress(proxyHost, proxyPort));
// Build HTTP Client
@@ -338,6 +354,98 @@ class FeedManagerImpl implements FeedManager, Service, Client {
return input.build(new XmlReader(stream));
}
+ private void postEntry(Feed feed, SyndEntry entry) {
+ LOG.info("Adding new entry...");
+
+ // build post body
+ StringBuilder b = new StringBuilder();
+ if (!StringUtils.isNullOrEmpty(entry.getTitle())) {
+ b.append(stripHTML(entry.getTitle())).append("\n\n");
+ }
+ for (SyndContent content : entry.getContents()) {
+ // extract content and do a very simple HTML tag stripping
+ if (content.getValue() != null)
+ b.append(stripHTML(content.getValue()));
+ }
+ if (entry.getContents().size() == 0) {
+ if (entry.getDescription().getValue() != null)
+ b.append(stripHTML(entry.getDescription().getValue()));
+ }
+ if (!StringUtils.isNullOrEmpty(entry.getAuthor())) {
+ b.append("\n\n-- ").append(stripHTML(entry.getAuthor()));
+ }
+ if (entry.getPublishedDate() != null) {
+ b.append(" (").append(entry.getPublishedDate().toString())
+ .append(")");
+ } else if (entry.getUpdatedDate() != null) {
+ b.append(" (").append(entry.getUpdatedDate().toString())
+ .append(")");
+ }
+ if (!StringUtils.isNullOrEmpty(entry.getLink())) {
+ b.append("\n\n").append(stripHTML(entry.getLink()));
+ }
+
+ // get other information for post
+ GroupId groupId = feed.getBlogId();
+ long time = clock.currentTimeMillis();
+ byte[] body = getPostBody(b.toString());
+ try {
+ // create and store post
+ Blog blog = blogManager.getBlog(groupId);
+ AuthorId authorId = blog.getAuthor().getId();
+ LocalAuthor author = identityManager.getLocalAuthor(authorId);
+ BlogPost post = blogPostFactory
+ .createBlogPost(groupId, null, time, null, author,
+ "text/plain", body);
+ blogManager.addLocalPost(post);
+ } catch (DbException e) {
+ if (LOG.isLoggable(WARNING))
+ LOG.log(WARNING, e.toString(), e);
+ } catch (GeneralSecurityException e) {
+ if (LOG.isLoggable(WARNING))
+ LOG.log(WARNING, e.toString(), e);
+ } catch (FormatException e) {
+ if (LOG.isLoggable(WARNING))
+ LOG.log(WARNING, e.toString(), e);
+ } catch (IllegalArgumentException e) {
+ // yes even catch this, so we at least get a stacktrace
+ // and the executor doesn't just die a silent death
+ if (LOG.isLoggable(WARNING))
+ LOG.log(WARNING, e.toString(), e);
+ }
+ }
+
+ private String stripHTML(String s) {
+ return StringUtils.trim(s.replaceAll("<.*?>", ""));
+ }
+
+ private byte[] getPostBody(String text) {
+ byte[] body = StringUtils.toUtf8(text);
+ if (body.length <= MAX_BLOG_POST_BODY_LENGTH) return body;
+ else return Arrays.copyOfRange(body, 0, MAX_BLOG_POST_BODY_LENGTH - 1);
+ }
+
+ /**
+ * This Comparator assumes that SyndEntry returns a valid Date either for
+ * getPublishedDate() or getUpdatedDate().
+ */
+ private Comparator<SyndEntry> getEntryComparator() {
+ return new Comparator<SyndEntry>() {
+ @Override
+ public int compare(SyndEntry e1, SyndEntry e2) {
+ Date d1 =
+ e1.getPublishedDate() != null ? e1.getPublishedDate() :
+ e1.getUpdatedDate();
+ Date d2 =
+ e2.getPublishedDate() != null ? e2.getPublishedDate() :
+ e2.getUpdatedDate();
+ if (d1.after(d2)) return 1;
+ if (d1.before(d2)) return -1;
+ return 0;
+ }
+ };
+ }
+
private Group getLocalGroup() {
return privateGroupFactory.createLocalGroup(getClientId());
}