tunnel quantity and length sliders step

Prep for push to transifex

See merge request zlatinb/muwire!47

.dockerignore

@@ -0,0 +1,12 @@
+# Dot directories
+.gradle/
+.idea/
+.git/
+
+# Build directories
+build/
+**/build/
+
+# We execute COPY . .
+# Modifying these files would unnecessarily invalidate the build context
+Dockerfile

.gitignore

@@ -2,7 +2,7 @@
 **/.settings
 **/build
 .gradle
-.project
-.classpath
+**/.project
+**/.classpath
 **/*.rej
 **/*.orig

Dockerfile

@@ -0,0 +1,64 @@
+FROM jlesage/baseimage-gui:alpine-3.10-glibc
+
+# Docker image version is provided via build arg.
+ARG DOCKER_IMAGE_VERSION=unknown
+
+# JDK version
+ARG JDK=11
+
+# Important directories
+ARG TMP_DIR=/muwire-tmp
+ENV APP_HOME=/muwire
+
+# Define working directory.
+WORKDIR $TMP_DIR
+
+# Put sources into dir
+COPY . .
+
+# Install final dependencies
+RUN add-pkg openjdk${JDK}-jre
+
+# Build and untar in future distribution dir
+RUN add-pkg --virtual openjdk${JDK}-jdk \
+        && ./gradlew --no-daemon clean assemble \
+        && mkdir -p ${APP_HOME} \
+        # Extract to ${APP_HOME and ignore the first dir
+        # First dir in tar is the "MuWire-<version>"
+        && tar -C ${APP_HOME} --strip 1 -xvf gui/build/distributions/MuWire*.tar \
+        # Cleanup
+        && rm -rf "${TMP_DIR}" /root/.gradle /root/.java \
+        && del-pkg openjdk${JDK}-jdk
+
+WORKDIR ${APP_HOME}
+
+# Maximize only the main/initial window.
+RUN \
+    sed-patch 's/<application type="normal">/<application type="normal" title="MuWire">/' \
+        /etc/xdg/openbox/rc.xml
+
+# Generate and install favicons.
+RUN \
+    APP_ICON_URL=https://github.com/zlatinb/muwire/raw/master/gui/griffon-app/resources/MuWire-128x128.png && \
+    install_app_icon.sh "$APP_ICON_URL"
+
+# Add files.
+COPY docker/rootfs/ /
+
+# Set environment variables.
+ENV APP_NAME="MuWire" \
+    S6_KILL_GRACETIME=8000
+
+# Define mountable directories.
+VOLUME ["$APP_HOME/.MuWire"]
+VOLUME ["/incompletes"]
+VOLUME ["/output"]
+
+
+# Metadata.
+LABEL \
+      org.label-schema.name="muwire" \
+      org.label-schema.description="Docker container for MuWire" \
+      org.label-schema.version="$DOCKER_IMAGE_VERSION" \
+      org.label-schema.vcs-url="https://github.com/zlatinb/muwire" \
+      org.label-schema.schema-version="1.0"

README.md

@@ -1,12 +1,14 @@
+The GitHub repo is mirrored from the in-I2P GitLab repo.  Please open PRs and issues at http://git.idk.i2p/zlatinb/muwire
+
 # MuWire - Easy Anonymous File-Sharing
 
 MuWire is an easy to use file-sharing program which offers anonymity using [I2P technology](http://geti2p.net).  It works on any platform Java works on, including Windows,MacOS,Linux.
 
-It is inspired by the LimeWire Gnutella client and developped by a former LimeWire developer.
+The current stable release - 0.6.8 is avaiable for download at https://muwire.com.  The latest plugin build and instructions how to install the plugin are available inside I2P at http://muwire.i2p.  
 
-The current stable release - 0.6.7 is avaiable for download at https://muwire.com.  You can find technical documentation in the [doc] folder.  Also check out the [Wiki] for various documentation.
+You can find technical documentation in the [doc] folder.  Also check out the [Wiki] for various other documentation.
 
-### Building
+## Building
 
 You need JDK 9 or newer.  After installing that and setting up the appropriate paths, just type
 
@@ -21,7 +23,7 @@ If you want to run the unit tests, type
 
 If you want to build binary bundles that do not depend on Java or I2P, see the [muwire-pkg] project
 
-### Running the GUI
+## Running the GUI
 
 Type
 ```
@@ -32,27 +34,30 @@ If you have an I2P router running on the same machine that is all you need to do
 
 [Default I2CP port]\: `7654`
 
-### Running the CLI
+## Running the CLI
 
 Look inside `cli-lanterna/build/distributions`.  Untar/unzip one of the `shadow` files and then run the jar contained inside by typing `java -jar cli-lanterna-x.y.z-all.jar` in a terminal.  The CLI will ask you about the router host and port on startup, no need to edit any files.  However, the CLI does not have an options window yet, so if you need to change any options you will need to edit the configuration files.  The CLI options are documented here [cli options]
 
 The CLI is under active development and doesn't have all the features of the GUI.
 
-### Running the Web UI / Plugin
+## Running the Web UI / Plugin
 
-There is a Web-based UI under development.  It is intended to be run as a plugin to the Java I2P router.  To build it, do the following:
-1. Check out the Java I2P code from here [I2P Github]
-2. Build it using `ant clean pkg`
-3. Create a directory under the MuWire source tree called `i2pjars` and copy all jars from the i2p `pkg-temp/lib` directory there.
-4. Un-comment the `webui` and `plug` projects in MuWire's `settings.gradle`.  (By default those are commented out)
-5. Edit `gradle.properties` to change the signer.  You need to sign your plugin for the I2P router to load it.  More information TODO
-6. Run `./gradlew -PbuildNumber=<integer> assemble`
-7. If all goes well, there will be a file `plug/build/MuWire.su3`.  Install that file from the I2P router console.
+There is a Web-based UI under development.  It is intended to be run as a plugin to the Java I2P router.  Instructions how to build it are available at the wiki [Plugin] page.
 
-### Translations
+## Docker
+
+MuWire is available as a Docker image.  For more information see the [Docker] page.
+
+## Translations
 If you want to help translate MuWire, instructions are on the wiki https://github.com/zlatinb/muwire/wiki/Translate
 
-### GPG Fingerprint
+## Related Projects
+### MuWire Tracker Daemon
+The MuWire Tracker Daemon (or mwtrackerd for short) is a project to bring functionality similar to BitTorrent tracking to MuWire.  For more info see the [Tracker] page.
+### MuCats
+MuCats is a project to create a website for hosting hashes of files shared on the MuWire network.  For more info see the [MuCats] project.
+
+## GPG Fingerprint
 
 ```
 471B 9FD4 5517 A5ED 101F  C57D A728 3207 2D52 5E41
@@ -67,3 +72,8 @@ You can find the full key at https://keybase.io/zlatinb
 [muwire-pkg]: https://github.com/zlatinb/muwire-pkg 
 [cli options]: https://github.com/zlatinb/muwire/wiki/CLI-Configuration-Options
 [I2P Github]: https://github.com/i2p/i2p.i2p
+[Plugin]: https://github.com/zlatinb/muwire/wiki/Plugin
+[Docker]: https://github.com/zlatinb/muwire/wiki/Docker
+[jlesage/docker-baseimage-gui]: https://github.com/jlesage/docker-baseimage-gui
+[Tracker]: https://github.com/zlatinb/muwire/wiki/Tracker-Daemon
+[MuCats]: https://github.com/zlatinb/mucats

TODO.md

@@ -1,8 +1,6 @@
 # TODO List
 
-Not in any particular order yet
-
-### Big Items
+### Network
 
 ##### Bloom Filters
 
@@ -12,15 +10,37 @@ This reduces query traffic by not sending last hop queries to peers that definit
 
 This helps with scalability
 
-##### Web UI, REST Interface, etc.
+### Core
 
-Basically any non-gui non-cli user interface
-
-##### Metadata editing and search
-
-To enable parsing of metadata from known file types and the user editing it or adding manual metadata
-
-### Small Items
-
-* Wrapper of some kind for in-place upgrades
+* Metadata parsing and search
 * Automatic adjustment of number of I2P tunnels
+* Persist trust immediately
+* Check if user-selected download and incomplete locations exist and are writeable
+* Enum i18n
+* Ability to share trust list only with trusted users
+* Confidential files visible only to certain users
+* Advertise file feed and browseability in upload headers
+* Manual polling / shared folder re-scan (because polling NAS doesn't work)
+
+### Chat
+* echo "unknown/innappropriate command" in the console
+* break up lines on CR/LF, send multiple messages
+* Style timestamps and persona names
+* enforce # in room names or ignore it
+* auto-create/join channel on server start
+* jump from notification window to room with message
+
+### Swing GUI
+* I2P Status panel - display message when connected to external router
+* Search box - left identation
+* Ability to disable switching of tabs on actions
+* Ability to trust/browse/subscribe from uploads tab
+
+### Web UI/Plugin
+* HTML 5 media players
+* Remove versions from jar names
+* Security: POST nonces, CSP headers
+* Upload files from browser to plugin via drag-and-drop
+* Check permissions, display better errors when sharing local folders
+
+ 

build.gradle

@@ -2,11 +2,26 @@ subprojects {
     apply plugin: 'groovy'
 
     dependencies {
-        compile 'org.codehaus.groovy:groovy-all:2.4.15'
+        compile 'org.codehaus.groovy:groovy:2.4.15'
+        compile 'org.codehaus.groovy:groovy-jsr223:2.4.15'
+        compile 'org.codehaus.groovy:groovy-json:2.4.15'
     }
 
     compileGroovy {
         groovyOptions.optimizationOptions.indy = true
+        sourceCompatibility = project.sourceCompatibility
+        targetCompatibility = project.targetCompatibility
+        options.compilerArgs += project.compilerArgs
+        options.deprecation = true
+        options.encoding = 'UTF-8'
+    }
+
+    compileJava {
+        sourceCompatibility = project.sourceCompatibility
+        targetCompatibility = project.targetCompatibility
+        options.compilerArgs += project.compilerArgs
+        options.deprecation = true
+        options.encoding = 'UTF-8'
     }
 
     repositories {

cli-lanterna/src/main/groovy/com/muwire/clilanterna/CliLanterna.groovy

@@ -32,7 +32,7 @@ import com.muwire.core.UILoadedEvent
 import com.muwire.core.files.AllFilesLoadedEvent
 
 class CliLanterna {
-    private static final String MW_VERSION = "0.6.8"
+    private static final String MW_VERSION = "0.7.0"
     
     private static volatile Core core
     

cli-lanterna/src/main/groovy/com/muwire/clilanterna/FilesModel.groovy

@@ -3,6 +3,7 @@ package com.muwire.clilanterna
 import com.googlecode.lanterna.gui2.TextGUIThread
 import com.googlecode.lanterna.gui2.table.TableModel
 import com.muwire.core.Core
+import com.muwire.core.InfoHash
 import com.muwire.core.SharedFile
 import com.muwire.core.files.AllFilesLoadedEvent
 import com.muwire.core.files.DirectoryWatchedEvent
@@ -27,7 +28,6 @@ class FilesModel {
         core.eventBus.register(FileLoadedEvent.class, this)
         core.eventBus.register(FileUnsharedEvent.class, this)
         core.eventBus.register(FileHashedEvent.class, this)
-        core.eventBus.register(AllFilesLoadedEvent.class, this)
         
         Runnable refreshModel = {refreshModel()}
         Timer timer = new Timer(true)
@@ -37,15 +37,6 @@ class FilesModel {
         
     }
     
-    void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
-        def eventBus = core.eventBus
-        guiThread.invokeLater {
-            core.muOptions.watchedDirectories.each {
-                eventBus.publish(new FileSharedEvent(file: new File(it)))
-            }
-        }
-    }
-    
     void onFileLoadedEvent(FileLoadedEvent e) {
         guiThread.invokeLater {
             sharedFiles.add(e.loadedFile)
@@ -72,7 +63,7 @@ class FilesModel {
         sharedFiles.each { 
             long size = it.getCachedLength()
             boolean comment = it.comment != null
-            boolean certified = core.certificateManager.hasLocalCertificate(it.getInfoHash())
+            boolean certified = core.certificateManager.hasLocalCertificate(new InfoHash(it.getRoot()))
             String hits = String.valueOf(it.getHits())
             String downloaders = String.valueOf(it.getDownloaders().size())
             model.addRow(new SharedFileWrapper(it), DataHelper.formatSize2(size, false)+"B", comment, certified, hits, downloaders)

cli-lanterna/src/main/groovy/com/muwire/clilanterna/FilesView.groovy

@@ -21,7 +21,6 @@ import com.muwire.core.filecert.UICreateCertificateEvent
 import com.muwire.core.files.DirectoryUnsharedEvent
 import com.muwire.core.files.FileSharedEvent
 import com.muwire.core.files.FileUnsharedEvent
-import com.muwire.core.files.UIPersistFilesEvent
 
 class FilesView extends BasicWindow {
     private final FilesModel model
@@ -84,7 +83,6 @@ class FilesView extends BasicWindow {
         
         Button unshareButton = new Button("Unshare", {
             core.eventBus.publish(new FileUnsharedEvent(unsharedFile : sf))
-            core.eventBus.publish(new UIPersistFilesEvent())
             MessageDialog.showMessageDialog(textGUI, "File Unshared", "Unshared "+sf.getFile().getName(), MessageDialogButton.OK)
         } )
         Button addCommentButton = new Button("Add Comment", {

core/build.gradle

@@ -1,12 +1,38 @@
-apply plugin : 'application'
-mainClassName = 'com.muwire.core.Core'
-applicationDefaultJvmArgs = ['-Djava.util.logging.config.file=logging.properties']
-dependencies {
-    compile "net.i2p:i2p:${i2pVersion}"
-    compile "net.i2p:router:${i2pVersion}"
-    compile "net.i2p.client:mstreaming:${i2pVersion}"
-    compile "net.i2p.client:streaming:${i2pVersion}"
 
-    testCompile 'org.junit.jupiter:junit-jupiter-api:5.4.2'    
-    testCompile 'junit:junit:4.12'
+plugins {
+    id 'java-library'
+    id 'maven-publish'
+}
+
+dependencies {
+    api "net.i2p:i2p:${i2pVersion}"
+    api "net.i2p:router:${i2pVersion}"
+    implementation "net.i2p.client:mstreaming:${i2pVersion}"
+    implementation "net.i2p.client:streaming:${i2pVersion}"
+
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2'    
+    testImplementation 'junit:junit:4.12'
+    testImplementation 'org.codehaus.groovy:groovy-all:2.4.15'
+}
+
+
+// this is necessary because applying both groovy and java-library doesn't work well
+configurations {
+    apiElements.outgoing.variants {
+        classes {
+            artifact file: compileGroovy.destinationDir, builtBy: compileGroovy
+        }
+    }
+}
+
+// publish core to local maven repo for sister projects
+publishing {
+    publications {
+        muCore(MavenPublication) {
+            from components.java
+        }
+    }
+    repositories {
+        mavenLocal()
+    }
 }

core/src/main/groovy/com/muwire/core/Core.groovy

@@ -1,7 +1,12 @@
 package com.muwire.core
 
+import com.muwire.core.files.PersisterDoneEvent
+import com.muwire.core.files.PersisterFolderService
+
 import java.nio.charset.StandardCharsets
 import java.util.concurrent.atomic.AtomicBoolean
+import java.util.logging.Level
+import java.util.zip.ZipException
 
 import com.muwire.core.chat.ChatDisconnectionEvent
 import com.muwire.core.chat.ChatManager
@@ -29,9 +34,18 @@ import com.muwire.core.filecert.CertificateManager
 import com.muwire.core.filecert.UICreateCertificateEvent
 import com.muwire.core.filecert.UIFetchCertificatesEvent
 import com.muwire.core.filecert.UIImportCertificateEvent
+import com.muwire.core.filefeeds.FeedClient
+import com.muwire.core.filefeeds.FeedFetchEvent
+import com.muwire.core.filefeeds.FeedItemFetchedEvent
+import com.muwire.core.filefeeds.FeedManager
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
+import com.muwire.core.filefeeds.UIFilePublishedEvent
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
+import com.muwire.core.filefeeds.UIFeedDeletedEvent
+import com.muwire.core.filefeeds.UIFeedUpdateEvent
+import com.muwire.core.filefeeds.UIFileUnpublishedEvent
 import com.muwire.core.files.FileDownloadedEvent
 import com.muwire.core.files.FileHashedEvent
-import com.muwire.core.files.FileHashingEvent
 import com.muwire.core.files.FileHasher
 import com.muwire.core.files.FileLoadedEvent
 import com.muwire.core.files.FileManager
@@ -41,7 +55,11 @@ import com.muwire.core.files.HasherService
 import com.muwire.core.files.PersisterService
 import com.muwire.core.files.SideCarFileEvent
 import com.muwire.core.files.UICommentEvent
-import com.muwire.core.files.UIPersistFilesEvent
+import com.muwire.core.files.directories.UISyncDirectoryEvent
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+import com.muwire.core.files.directories.WatchedDirectoryConvertedEvent
+import com.muwire.core.files.directories.WatchedDirectoryConverter
+import com.muwire.core.files.directories.WatchedDirectoryManager
 import com.muwire.core.files.AllFilesLoadedEvent
 import com.muwire.core.files.DirectoryUnsharedEvent
 import com.muwire.core.files.DirectoryWatchedEvent
@@ -67,6 +85,7 @@ import com.muwire.core.upload.UploadManager
 import com.muwire.core.util.MuWireLogManager
 import com.muwire.core.content.ContentControlEvent
 import com.muwire.core.content.ContentManager
+import com.muwire.core.tracker.TrackerResponder
 
 import groovy.util.logging.Log
 import net.i2p.I2PAppContext
@@ -74,10 +93,8 @@ import net.i2p.client.I2PClientFactory
 import net.i2p.client.I2PSession
 import net.i2p.client.streaming.I2PSocketManager
 import net.i2p.client.streaming.I2PSocketManagerFactory
-import net.i2p.client.streaming.I2PSocketOptions
 import net.i2p.client.streaming.I2PSocketManager.DisconnectListener
 import net.i2p.crypto.DSAEngine
-import net.i2p.crypto.SigType
 import net.i2p.data.Destination
 import net.i2p.data.PrivateKey
 import net.i2p.data.Signature
@@ -96,18 +113,19 @@ public class Core {
     final Properties i2pOptions
     final MuWireSettings muOptions
 
-    private final I2PSession i2pSession;
+    final I2PSession i2pSession;
     final TrustService trustService
     final TrustSubscriber trustSubscriber
     private final PersisterService persisterService
-    private final HostCache hostCache
-    private final ConnectionManager connectionManager
+    private final PersisterFolderService persisterFolderService
+    final HostCache hostCache
+    final ConnectionManager connectionManager
     private final CacheClient cacheClient
     private final UpdateClient updateClient
-    private final ConnectionAcceptor connectionAcceptor
+    final ConnectionAcceptor connectionAcceptor
     private final ConnectionEstablisher connectionEstablisher
     private final HasherService hasherService
-    private final DownloadManager downloadManager
+    final DownloadManager downloadManager
     private final DirectoryWatcher directoryWatcher
     final FileManager fileManager
     final UploadManager uploadManager
@@ -115,6 +133,11 @@ public class Core {
     final CertificateManager certificateManager
     final ChatServer chatServer
     final ChatManager chatManager
+    final FeedManager feedManager
+    private final FeedClient feedClient
+    private final WatchedDirectoryConverter watchedDirectoryConverter
+    final WatchedDirectoryManager watchedDirectoryManager
+    private final TrackerResponder trackerResponder
 
     private final Router router
 
@@ -128,29 +151,35 @@ public class Core {
         this.muOptions = props
 
         i2pOptions = new Properties()
-        def i2pOptionsFile = new File(home,"i2p.properties")
+        // Read defaults
+        def defaultI2PFile = getClass()
+                .getClassLoader().getResource("defaults/i2p.properties");
+        try {
+            defaultI2PFile.withInputStream { i2pOptions.load(it) }
+        } catch (ZipException mystery) {
+            log.log(Level.SEVERE, "couldn't load default i2p properties", mystery)
+        }
+
+        def i2pOptionsFile = new File(home, "i2p.properties")
         if (i2pOptionsFile.exists()) {
             i2pOptionsFile.withInputStream { i2pOptions.load(it) }
 
             if (!i2pOptions.containsKey("inbound.nickname"))
                 i2pOptions["inbound.nickname"] = "MuWire"
-                if (!i2pOptions.containsKey("outbound.nickname"))
-                    i2pOptions["outbound.nickname"] = "MuWire"
-        } else {
-            i2pOptions["inbound.nickname"] = "MuWire"
-            i2pOptions["outbound.nickname"] = "MuWire"
-            i2pOptions["inbound.length"] = "3"
-            i2pOptions["inbound.quantity"] = "4"
-            i2pOptions["outbound.length"] = "3"
-            i2pOptions["outbound.quantity"] = "4"
-            i2pOptions["i2cp.tcp.host"] = "127.0.0.1"
-            i2pOptions["i2cp.tcp.port"] = "7654"
+            if (!i2pOptions.containsKey("outbound.nickname"))
+                i2pOptions["outbound.nickname"] = "MuWire"
+        }
+        if (!(i2pOptions.containsKey("i2np.ntcp.port")
+                && i2pOptions.containsKey("i2np.udp.port")
+        )) {
             Random r = new Random()
-            int port = r.nextInt(60000) + 4000
+            int port = 9151 + r.nextInt(1 + 30777 - 9151)  // this range matches what the i2p router would choose
             i2pOptions["i2np.ntcp.port"] = String.valueOf(port)
             i2pOptions["i2np.udp.port"] = String.valueOf(port)
             i2pOptionsFile.withOutputStream { i2pOptions.store(it, "") }
         }
+        
+        i2pOptions['i2cp.leaseSetEncType']='4,0'
 
         if (!props.embeddedRouter) {
             if (!(I2PAppContext.getGlobalContext() instanceof RouterContext)) {
@@ -193,7 +222,7 @@ public class Core {
         // options like tunnel length and quantity
         I2PSocketManager socketManager
         keyDat.withInputStream {
-            socketManager = new I2PSocketManagerFactory().createManager(it, i2pOptions["i2cp.tcp.host"], i2pOptions["i2cp.tcp.port"].toInteger(), i2pOptions)
+            socketManager = new I2PSocketManagerFactory().createDisconnectedManager(it, i2pOptions["i2cp.tcp.host"], i2pOptions["i2cp.tcp.port"].toInteger(), i2pOptions)
         }
         socketManager.getDefaultOptions().setReadTimeout(60000)
         socketManager.getDefaultOptions().setConnectTimeout(30000)
@@ -259,7 +288,17 @@ public class Core {
         log.info "initializing persistence service"
         persisterService = new PersisterService(new File(home, "files.json"), eventBus, 60000, fileManager)
         eventBus.register(UILoadedEvent.class, persisterService)
-        eventBus.register(UIPersistFilesEvent.class, persisterService)
+
+        log.info "initializing folder persistence service"
+        persisterFolderService = new PersisterFolderService(this, new File(home, "files"), eventBus)
+        eventBus.register(PersisterDoneEvent.class, persisterFolderService)
+        eventBus.register(FileDownloadedEvent.class, persisterFolderService)
+        eventBus.register(FileLoadedEvent.class, persisterFolderService)
+        eventBus.register(FileHashedEvent.class, persisterFolderService)
+        eventBus.register(FileUnsharedEvent.class, persisterFolderService)
+        eventBus.register(UICommentEvent.class, persisterFolderService)
+        eventBus.register(UIFilePublishedEvent.class, persisterFolderService)
+        eventBus.register(UIFileUnpublishedEvent.class, persisterFolderService)
 
         log.info("initializing host cache")
         File hostStorage = new File(home, "hosts.json")
@@ -279,10 +318,13 @@ public class Core {
         log.info("initializing cache client")
         cacheClient = new CacheClient(eventBus,hostCache, connectionManager, i2pSession, props, 10000)
 
+        if (!props.plugin) {
         log.info("initializing update client")
-        updateClient = new UpdateClient(eventBus, i2pSession, myVersion, props, fileManager, me, spk)
-        eventBus.register(FileDownloadedEvent.class, updateClient)
-        eventBus.register(UIResultBatchEvent.class, updateClient)
+            updateClient = new UpdateClient(eventBus, i2pSession, myVersion, props, fileManager, me, spk)
+            eventBus.register(FileDownloadedEvent.class, updateClient)
+            eventBus.register(UIResultBatchEvent.class, updateClient)
+        } else
+            log.info("running as plugin, not initializing update client")
 
         log.info("initializing connector")
         I2PConnector i2pConnector = new I2PConnector(socketManager)
@@ -299,6 +341,19 @@ public class Core {
             register(TrustEvent.class, chatServer)
         }
         
+        log.info("initializing feed manager")
+        feedManager = new FeedManager(eventBus, home)
+        eventBus.with { 
+            register(FeedItemFetchedEvent.class, feedManager)
+            register(FeedFetchEvent.class, feedManager)
+            register(UIFeedConfigurationEvent.class, feedManager)
+            register(UIFeedDeletedEvent.class, feedManager)
+        }
+        
+        log.info("initializing feed client")
+        feedClient = new FeedClient(i2pConnector, eventBus, me, feedManager)
+        eventBus.register(UIFeedUpdateEvent.class, feedClient)
+        
         log.info "initializing results sender"
         ResultsSender resultsSender = new ResultsSender(eventBus, i2pConnector, me, props, certificateManager, chatServer)
 
@@ -310,6 +365,7 @@ public class Core {
         log.info("initializing download manager")
         downloadManager = new DownloadManager(eventBus, trustService, meshManager, props, i2pConnector, home, me)
         eventBus.register(UIDownloadEvent.class, downloadManager)
+        eventBus.register(UIDownloadFeedItemEvent.class, downloadManager)
         eventBus.register(UILoadedEvent.class, downloadManager)
         eventBus.register(FileDownloadedEvent.class, downloadManager)
         eventBus.register(UIDownloadCancelledEvent.class, downloadManager)
@@ -318,7 +374,10 @@ public class Core {
         eventBus.register(UIDownloadResumedEvent.class, downloadManager)
 
         log.info("initializing upload manager")
-        uploadManager = new UploadManager(eventBus, fileManager, meshManager, downloadManager, props)
+        uploadManager = new UploadManager(eventBus, fileManager, meshManager, downloadManager, persisterFolderService, props)
+        
+        log.info("initializing tracker responder")
+        trackerResponder = new TrackerResponder(i2pSession, props, fileManager, downloadManager, meshManager, trustService, me)
 
         log.info("initializing connection establisher")
         connectionEstablisher = new ConnectionEstablisher(eventBus, i2pConnector, props, connectionManager, hostCache)
@@ -339,11 +398,6 @@ public class Core {
             i2pAcceptor, hostCache, trustService, searchManager, uploadManager, fileManager, connectionEstablisher,
             certificateManager, chatServer)
 
-        log.info("initializing directory watcher")
-        directoryWatcher = new DirectoryWatcher(eventBus, fileManager, home, props)
-        eventBus.register(DirectoryWatchedEvent.class, directoryWatcher)
-        eventBus.register(AllFilesLoadedEvent.class, directoryWatcher)
-        eventBus.register(DirectoryUnsharedEvent.class, directoryWatcher)
 
         log.info("initializing hasher service")
         hasherService = new HasherService(new FileHasher(), eventBus, fileManager, props)
@@ -365,9 +419,32 @@ public class Core {
         BrowseManager browseManager = new BrowseManager(i2pConnector, eventBus, me)
         eventBus.register(UIBrowseEvent.class, browseManager)
         
+        log.info("initializing watched directory converter")
+        watchedDirectoryConverter = new WatchedDirectoryConverter(this)
+        eventBus.register(AllFilesLoadedEvent.class, watchedDirectoryConverter)
+        
+        log.info("initializing watched directory manager")
+        watchedDirectoryManager = new WatchedDirectoryManager(home, eventBus, fileManager)
+        eventBus.with { 
+            register(WatchedDirectoryConfigurationEvent.class, watchedDirectoryManager)
+            register(WatchedDirectoryConvertedEvent.class, watchedDirectoryManager)
+            register(FileSharedEvent.class, watchedDirectoryManager)
+            register(DirectoryUnsharedEvent.class, watchedDirectoryManager)
+            register(UISyncDirectoryEvent.class, watchedDirectoryManager)
+        }
+        
+        log.info("initializing directory watcher")
+        directoryWatcher = new DirectoryWatcher(eventBus, fileManager, home, watchedDirectoryManager)
+        eventBus.with {
+            register(DirectoryWatchedEvent.class, directoryWatcher)
+            register(WatchedDirectoryConvertedEvent.class, directoryWatcher)
+            register(DirectoryUnsharedEvent.class, directoryWatcher)
+            register(WatchedDirectoryConfigurationEvent.class, directoryWatcher)
+        }
     }
 
     public void startServices() {
+        i2pSession.connect()
         hasherService.start()
         trustService.start()
         trustService.waitForLoad()
@@ -377,7 +454,10 @@ public class Core {
         connectionAcceptor.start()
         connectionEstablisher.start()
         hostCache.waitForLoad()
-        updateClient.start()
+        updateClient?.start()
+        feedManager.start()
+        feedClient.start()
+        trackerResponder.start()
     }
 
     public void shutdown() {
@@ -387,12 +467,16 @@ public class Core {
         }
         log.info("saving settings")
         saveMuSettings()
+        log.info("shutting down host cache")
+        hostCache.stop()
         log.info("shutting down trust subscriber")
         trustSubscriber.stop()
         log.info("shutting down trust service")
         trustService.stop()
         log.info("shutting down persister service")
         persisterService.stop()
+        log.info("shutting down persisterFolder service")
+        persisterFolderService.stop()
         log.info("shutting down download manager")
         downloadManager.shutdown()
         log.info("shutting down connection acceptor")
@@ -401,12 +485,20 @@ public class Core {
         connectionEstablisher.stop()
         log.info("shutting down directory watcher")
         directoryWatcher.stop()
+        log.info("shutting down watch directory manager")
+        watchedDirectoryManager.shutdown()
         log.info("shutting down cache client")
         cacheClient.stop()
         log.info("shutting down chat server")
         chatServer.stop()
         log.info("shutting down chat manager")
         chatManager.shutdown()
+        log.info("shutting down feed manager")
+        feedManager.stop()
+        log.info("shutting down feed client")
+        feedClient.stop()
+        log.info("shutting down tracker responder")
+        trackerResponder.stop()
         log.info("shutting down connection manager")
         connectionManager.shutdown()
         log.info("killing i2p session")
@@ -454,7 +546,7 @@ public class Core {
             }
         }
 
-        Core core = new Core(props, home, "0.6.8")
+        Core core = new Core(props, home, "0.7.0")
         core.startServices()
 
         // ... at the end, sleep or execute script

core/src/main/groovy/com/muwire/core/MuWireSettings.groovy

@@ -31,6 +31,17 @@ class MuWireSettings {
     boolean shareHiddenFiles
     boolean searchComments
     boolean browseFiles
+    boolean allowTracking
+    
+    boolean fileFeed
+    boolean advertiseFeed
+    boolean autoPublishSharedFiles
+    boolean defaultFeedAutoDownload
+    int defaultFeedUpdateInterval
+    int defaultFeedItemsToKeep
+    boolean defaultFeedSequential
+    
+    
     boolean startChatServer
     int maxChatConnections
     boolean advertiseChat
@@ -41,6 +52,7 @@ class MuWireSettings {
     int meshExpiration
     int speedSmoothSeconds
     boolean embeddedRouter
+    boolean plugin
     int inBw, outBw
     Set<String> watchedKeywords
     Set<String> watchedRegexes
@@ -76,10 +88,22 @@ class MuWireSettings {
         hostRejectInterval = Integer.valueOf(props.getProperty("hostRejectInterval", "1"))
         meshExpiration = Integer.valueOf(props.getProperty("meshExpiration","60"))
         embeddedRouter = Boolean.valueOf(props.getProperty("embeddedRouter","false"))
+        plugin = Boolean.valueOf(props.getProperty("plugin","false"))
         inBw = Integer.valueOf(props.getProperty("inBw","256"))
         outBw = Integer.valueOf(props.getProperty("outBw","128"))
         searchComments = Boolean.valueOf(props.getProperty("searchComments","true"))
         browseFiles = Boolean.valueOf(props.getProperty("browseFiles","true"))
+        allowTracking = Boolean.valueOf(props.getProperty("allowTracking","true"))
+        
+        // feed settings
+        fileFeed = Boolean.valueOf(props.getProperty("fileFeed","true"))
+        advertiseFeed = Boolean.valueOf(props.getProperty("advertiseFeed","true"))
+        autoPublishSharedFiles = Boolean.valueOf(props.getProperty("autoPublishSharedFiles", "false"))
+        defaultFeedAutoDownload = Boolean.valueOf(props.getProperty("defaultFeedAutoDownload", "false"))
+        defaultFeedItemsToKeep = Integer.valueOf(props.getProperty("defaultFeedItemsToKeep", "1000"))
+        defaultFeedSequential = Boolean.valueOf(props.getProperty("defaultFeedSequential", "false"))
+        defaultFeedUpdateInterval = Integer.valueOf(props.getProperty("defaultFeedUpdateInterval", "60000"))
+        
         speedSmoothSeconds = Integer.valueOf(props.getProperty("speedSmoothSeconds","60"))
         totalUploadSlots = Integer.valueOf(props.getProperty("totalUploadSlots","-1"))
         uploadSlotsPerUser = Integer.valueOf(props.getProperty("uploadSlotsPerUser","-1"))
@@ -130,10 +154,22 @@ class MuWireSettings {
         props.setProperty("hostRejectInterval", String.valueOf(hostRejectInterval))
         props.setProperty("meshExpiration", String.valueOf(meshExpiration))
         props.setProperty("embeddedRouter", String.valueOf(embeddedRouter))
+        props.setProperty("plugin", String.valueOf(plugin))
         props.setProperty("inBw", String.valueOf(inBw))
         props.setProperty("outBw", String.valueOf(outBw))
         props.setProperty("searchComments", String.valueOf(searchComments))
         props.setProperty("browseFiles", String.valueOf(browseFiles))
+        props.setProperty("allowTracking", String.valueOf(allowTracking))
+        
+        // feed settings
+        props.setProperty("fileFeed", String.valueOf(fileFeed))
+        props.setProperty("advertiseFeed", String.valueOf(advertiseFeed))
+        props.setProperty("autoPublishSharedFiles", String.valueOf(autoPublishSharedFiles))
+        props.setProperty("defaultFeedAutoDownload", String.valueOf(defaultFeedAutoDownload))
+        props.setProperty("defaultFeedItemsToKeep", String.valueOf(defaultFeedItemsToKeep))
+        props.setProperty("defaultFeedSequential", String.valueOf(defaultFeedSequential))
+        props.setProperty("defaultFeedUpdateInterval", String.valueOf(defaultFeedUpdateInterval))
+        
         props.setProperty("speedSmoothSeconds", String.valueOf(speedSmoothSeconds))
         props.setProperty("totalUploadSlots", String.valueOf(totalUploadSlots))
         props.setProperty("uploadSlotsPerUser", String.valueOf(uploadSlotsPerUser))

core/src/main/groovy/com/muwire/core/chat/ChatConnection.groovy

@@ -233,7 +233,7 @@ class ChatConnection implements ChatLink {
         daos.close()
         byte [] signed = baos.toByteArray()
         def spk = sender.destination.getSigningPublicKey()
-        def signature = new Signature(Constants.SIG_TYPE, sig)
+        def signature = new Signature(spk.getType(), sig)
         DSAEngine.getInstance().verifySignature(signature, signed, spk)
     }
     

core/src/main/groovy/com/muwire/core/connection/Connection.groovy

@@ -244,7 +244,7 @@ abstract class Connection implements Closeable {
             else 
                 payload =  String.join(" ",search.keywords).getBytes(StandardCharsets.UTF_8)
             def spk = originator.destination.getSigningPublicKey()
-            def signature = new Signature(Constants.SIG_TYPE, sig)
+            def signature = new Signature(spk.getType(), sig)
             if (!DSAEngine.getInstance().verifySignature(signature, payload, spk)) {
                 log.info("signature didn't match keywords")
                 return
@@ -255,7 +255,6 @@ abstract class Connection implements Closeable {
             return
         }
         
-        // TODO: make this mandatory at some point
         byte[] sig2 = null        
         long queryTime = 0
         if (search.sig2 != null) {
@@ -267,7 +266,7 @@ abstract class Connection implements Closeable {
             queryTime = search.queryTime
             byte [] payload = (search.uuid + String.valueOf(queryTime)).getBytes(StandardCharsets.US_ASCII)
             def spk = originator.destination.getSigningPublicKey()
-            def signature = new Signature(Constants.SIG_TYPE, sig2)
+            def signature = new Signature(spk.getType(), sig2)
             if (!DSAEngine.getInstance().verifySignature(signature, payload, spk)) {
                 log.info("extended signature didn't match uuid and timestamp")
                 return
@@ -278,8 +277,10 @@ abstract class Connection implements Closeable {
                     return
                 }
             }
-        } else
+        } else {
             log.info("no extended signature in query")
+            return
+        }
 
         SearchEvent searchEvent = new SearchEvent(searchTerms : search.keywords,
                                             searchHash : infohash,

core/src/main/groovy/com/muwire/core/connection/ConnectionAcceptor.groovy

@@ -15,9 +15,11 @@ import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
 import com.muwire.core.Persona
+import com.muwire.core.SharedFile
 import com.muwire.core.chat.ChatServer
 import com.muwire.core.filecert.Certificate
 import com.muwire.core.filecert.CertificateManager
+import com.muwire.core.filefeeds.FeedItems
 import com.muwire.core.files.FileManager
 import com.muwire.core.hostcache.HostCache
 import com.muwire.core.trust.TrustLevel
@@ -58,7 +60,7 @@ class ConnectionAcceptor {
 
     private volatile shutdown
     
-    private volatile int browsed
+    volatile int browsed
 
     ConnectionAcceptor(EventBus eventBus, UltrapeerConnectionManager manager,
         MuWireSettings settings, I2PAcceptor acceptor, HostCache hostCache,
@@ -161,6 +163,9 @@ class ConnectionAcceptor {
                 case (byte)'I':
                     processIRC(e)
                     break
+                case (byte)'F':
+                    processFEED(e)
+                    break
                 default:
                     throw new Exception("Invalid read $read")
             }
@@ -310,6 +315,9 @@ class ConnectionAcceptor {
             boolean chat = false
             if (headers.containsKey('Chat'))
                 chat = Boolean.parseBoolean(headers['Chat'])
+            boolean feed = false
+            if (headers.containsKey('Feed'))
+                feed = Boolean.parseBoolean(headers['Feed'])
                 
             byte [] personaBytes = Base64.decode(headers['Sender'])
             Persona sender = new Persona(new ByteArrayInputStream(personaBytes))
@@ -329,6 +337,7 @@ class ConnectionAcceptor {
                 def json = slurper.parse(payload)
                 results[i] = ResultsParser.parse(sender, resultsUUID, json)
                 results[i].chat = chat
+                results[i].feed = feed
             }
             eventBus.publish(new UIResultBatchEvent(uuid: resultsUUID, results: results))
         } catch (IOException bad) {
@@ -374,13 +383,16 @@ class ConnectionAcceptor {
             boolean chat = chatServer.running.get() && settings.advertiseChat
             os.write("Chat: ${chat}\r\n".getBytes(StandardCharsets.US_ASCII))
             
+            boolean feed = settings.fileFeed && settings.advertiseFeed
+            os.write("Feed: ${feed}\r\n".getBytes(StandardCharsets.US_ASCII))
+            
             os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
 
             DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))
             JsonOutput jsonOutput = new JsonOutput()
             sharedFiles.each {
                 it.hit(browser, System.currentTimeMillis(), "Browse Host");
-                int certificates = certificateManager.getByInfoHash(it.getInfoHash()).size()
+                int certificates = certificateManager.getByInfoHash(new InfoHash(it.getRoot())).size()
                 def obj = ResultsSender.sharedFileToObj(it, false, certificates)
                 def json = jsonOutput.toJson(obj)
                 dos.writeShort((short)json.length())
@@ -524,5 +536,58 @@ class ConnectionAcceptor {
             throw new Exception("Invalid IRC connection")
         chatServer.handle(e)
     }
+    
+    private void processFEED(Endpoint e) {
+        try {
+            byte[] EED = new byte[5];
+            DataInputStream dis = new DataInputStream(e.getInputStream())
+            dis.readFully(EED);
+            if (EED != "EED\r\n".getBytes(StandardCharsets.US_ASCII))
+                throw new Exception("Invalid FEED connection")
+
+            OutputStream os = e.getOutputStream()
+
+            Map<String, String> headers = DataUtil.readAllHeaders(dis)
+            if (!headers.containsKey("Persona"))
+                throw new Exception("Persona header missing")
+            Persona requestor = new Persona(new ByteArrayInputStream(Base64.decode(headers['Persona'])))
+            if (requestor.destination != e.destination)
+                throw new Exception("Requestor persona mismatch")
+
+            if (!settings.fileFeed) {
+                os.write("403 Not Allowed\r\n\r\n".getBytes(StandardCharsets.US_ASCII))
+                os.flush()
+                e.close()
+                return
+            }
+
+            long timestamp = 0
+            if (headers.containsKey("Timestamp")) {
+                timestamp = Long.parseLong(headers['Timestamp'])
+            }
+
+            List<SharedFile> published = fileManager.getPublishedSince(timestamp)
+
+            os.write("200 OK\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("Count: ${published.size()}\r\n".getBytes(StandardCharsets.US_ASCII));
+            os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
+
+            DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))
+            JsonOutput jsonOutput = new JsonOutput()
+            final long now = System.currentTimeMillis();
+            published.each {
+                it.hit(requestor, now, "Feed Update");
+                int certificates = certificateManager.getByInfoHash(new InfoHash(it.getRoot())).size()
+                def obj = FeedItems.sharedFileToObj(it, certificates)
+                def json = jsonOutput.toJson(obj)
+                dos.writeShort((short)json.length())
+                dos.write(json.getBytes(StandardCharsets.US_ASCII))
+            }
+            dos.flush()
+            dos.close()
+        } finally {
+            e.close()
+        }
+    }
 
 }

core/src/main/groovy/com/muwire/core/download/DownloadManager.groovy

@@ -1,6 +1,7 @@
 package com.muwire.core.download
 
 import com.muwire.core.connection.I2PConnector
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
 import com.muwire.core.files.FileDownloadedEvent
 import com.muwire.core.files.FileHasher
 import com.muwire.core.mesh.Mesh
@@ -62,11 +63,6 @@ public class DownloadManager {
 
 
     public void onUIDownloadEvent(UIDownloadEvent e) {
-
-        File incompletes = muSettings.incompleteLocation
-        if (incompletes == null)
-            incompletes = new File(home, "incompletes")
-        incompletes.mkdirs()
         
         def size = e.result[0].size
         def infohash = e.result[0].infohash
@@ -79,12 +75,29 @@ public class DownloadManager {
         destinations.addAll(e.sources)
         destinations.remove(me.destination)
 
-        Pieces pieces = getPieces(infohash, size, pieceSize, e.sequential)
+        doDownload(infohash, e.target, size, pieceSize, e.sequential, destinations)
 
-        def downloader = new Downloader(eventBus, this, me, e.target, size,
-            infohash, pieceSize, connector, destinations,
-            incompletes, pieces)
-        downloaders.put(infohash, downloader)
+    }
+    
+    public void onUIDownloadFeedItemEvent(UIDownloadFeedItemEvent e) {
+        Set<Destination> singleSource = new HashSet<>()
+        singleSource.add(e.item.getPublisher().getDestination())
+        doDownload(e.item.getInfoHash(), e.target, e.item.getSize(), e.item.getPieceSize(), 
+            e.sequential, singleSource)
+    }
+    
+    private void doDownload(InfoHash infoHash, File target, long size, int pieceSize, 
+        boolean sequential, Set<Destination> destinations) {
+        File incompletes = muSettings.incompleteLocation
+        if (incompletes == null)
+            incompletes = new File(home, "incompletes")
+        incompletes.mkdirs()
+        
+        Pieces pieces = getPieces(infoHash, size, pieceSize, sequential)
+        def downloader = new Downloader(eventBus, this, me, target, size,
+                infoHash, pieceSize, connector, destinations,
+                incompletes, pieces)
+        downloaders.put(infoHash, downloader)
         persistDownloaders()
         executor.execute({downloader.download()} as Runnable)
         eventBus.publish(new DownloadStartedEvent(downloader : downloader))
@@ -229,4 +242,8 @@ public class DownloadManager {
         downloaders.values().each { it.stop() }
         Downloader.executorService.shutdownNow()
     }
+    
+    public boolean isDownloading(InfoHash infoHash) {
+        downloaders.containsKey(infoHash)
+    }
 }

core/src/main/groovy/com/muwire/core/download/DownloadSession.groovy

@@ -183,15 +183,14 @@ class DownloadSession {
                 mapped.position(position)
 
                 byte[] tmp = new byte[0x1 << 13]
+                DataInputStream dis = new DataInputStream(is)
                 while(mapped.hasRemaining()) {
                     if (mapped.remaining() < tmp.length)
                         tmp = new byte[mapped.remaining()]
-                    int read = is.read(tmp)
-                    if (read == -1)
-                        throw new IOException()
+                    dis.readFully(tmp)
                     synchronized(this) {
-                        mapped.put(tmp, 0, read)
-                        dataSinceLastRead.addAndGet(read)
+                        mapped.put(tmp)
+                        dataSinceLastRead.addAndGet(tmp.length)
                         pieces.markPartial(piece, mapped.position())
                     }
                 }

core/src/main/groovy/com/muwire/core/download/Downloader.groovy

@@ -160,7 +160,7 @@ public class Downloader {
             long dataRead = dataSinceLastRead.getAndSet(0)
             long now = System.currentTimeMillis()
             if (now > lastSpeedRead)
-                currSpeed = (int) (dataRead * 1000.0 / (now - lastSpeedRead))
+                currSpeed = (int) (dataRead * 1000.0d / (now - lastSpeedRead))
             lastSpeedRead = now
         }
         
@@ -405,8 +405,9 @@ public class Downloader {
                     }
                     eventBus.publish(
                         new FileDownloadedEvent(
-                            downloadedFile : new DownloadedFile(file.getCanonicalFile(), getInfoHash(), pieceSizePow2, successfulDestinations),
-                        downloader : Downloader.this))
+                            downloadedFile : new DownloadedFile(file.getCanonicalFile(), getInfoHash().getRoot(), pieceSizePow2, successfulDestinations),
+                        downloader : Downloader.this,
+                        infoHash: getInfoHash()))
 
                 }
                 endpoint?.close()

core/src/main/groovy/com/muwire/core/download/Pieces.groovy

@@ -2,10 +2,11 @@ package com.muwire.core.download
 
 class Pieces {
     private final BitSet done, claimed
-    private final int nPieces
+    final int nPieces
     private final float ratio
     private final Random random = new Random()
     private final Map<Integer,Integer> partials = new HashMap<>()
+    private int cachedDone;
 
     Pieces(int nPieces) {
         this(nPieces, 1.0f)
@@ -78,6 +79,7 @@ class Pieces {
         if (piece >= nPieces)
             throw new IllegalArgumentException("invalid piece marked as downloaded? $piece/$nPieces")
         done.set(piece)
+        cachedDone = done.cardinality();
         claimed.set(piece)
         partials.remove(piece)
     }
@@ -91,11 +93,11 @@ class Pieces {
     }
 
     synchronized boolean isComplete() {
-        done.cardinality() == nPieces
+        cachedDone == nPieces
     }
 
     synchronized int donePieces() {
-        done.cardinality()
+        cachedDone
     }
 
     synchronized boolean isDownloaded(int piece) {
@@ -104,6 +106,7 @@ class Pieces {
 
     synchronized void clearAll() {
         done.clear()
+        cachedDone = 0
         claimed.clear()
         partials.clear()
     }

core/src/main/groovy/com/muwire/core/filecert/Certificate.groovy

@@ -105,7 +105,7 @@ class Certificate {
         
         byte [] payload = baos.toByteArray()
         SigningPublicKey spk = issuer.destination.getSigningPublicKey()
-        Signature signature = new Signature(Constants.SIG_TYPE, sig)
+        Signature signature = new Signature(spk.getType(), sig)
         DSAEngine.getInstance().verifySignature(signature, payload, spk)
     }
     

core/src/main/groovy/com/muwire/core/filecert/CertificateManager.groovy

@@ -70,7 +70,7 @@ class CertificateManager {
     }
     
     void onUICreateCertificateEvent(UICreateCertificateEvent e) {
-        InfoHash infoHash = e.sharedFile.getInfoHash()
+        InfoHash infoHash = new InfoHash(e.sharedFile.getRoot())
         String name = e.sharedFile.getFile().getName()
         long timestamp = System.currentTimeMillis()
         

core/src/main/groovy/com/muwire/core/filefeeds/FeedClient.groovy

@@ -0,0 +1,110 @@
+package com.muwire.core.filefeeds
+
+import java.util.logging.Level
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.Executor
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.zip.GZIPInputStream
+
+import com.muwire.core.EventBus
+import com.muwire.core.Persona
+import com.muwire.core.connection.Endpoint
+import com.muwire.core.connection.I2PConnector
+import com.muwire.core.util.DataUtil
+
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+@Log
+class FeedClient {
+    
+    private final I2PConnector connector
+    private final EventBus eventBus
+    private final Persona me
+    private final FeedManager feedManager
+    
+    private final ExecutorService feedFetcher = Executors.newCachedThreadPool() 
+    private final Timer feedUpdater = new Timer("feed-updater", true)
+    
+    FeedClient(I2PConnector connector, EventBus eventBus, Persona me, FeedManager feedManager) {
+        this.connector = connector
+        this.eventBus = eventBus
+        this.me = me
+        this.feedManager = feedManager
+    }
+    
+    private void start() {
+        feedUpdater.schedule({updateAnyFeeds()} as TimerTask, 60000, 60000)
+    }
+    
+    private void stop() {
+        feedUpdater.cancel()
+        feedFetcher.shutdown()
+    }
+    
+    private void updateAnyFeeds() {
+        feedManager.getFeedsToUpdate().each { feed ->
+            feedFetcher.execute({updateFeed(feed)} as Runnable)
+        }
+    }
+    
+    void onUIFeedUpdateEvent(UIFeedUpdateEvent e) {
+        Feed feed = feedManager.getFeed(e.host)
+        if (feed == null) {
+            log.severe("UI request to update non-existent feed " + e.host.getHumanReadableName())
+            return
+        }
+        
+        feedFetcher.execute({updateFeed(feed)} as Runnable)
+    }
+    
+    private void updateFeed(Feed feed) {
+        log.info("updating feed " + feed.getPublisher().getHumanReadableName())
+        Endpoint endpoint = null
+        try {
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.CONNECTING))
+            feed.setLastUpdateAttempt(System.currentTimeMillis())
+            endpoint = connector.connect(feed.getPublisher().getDestination())
+            OutputStream os = endpoint.getOutputStream()
+            os.write("FEED\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("Persona:${me.toBase64()}\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("Timestamp:${feed.getLastUpdated()}\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
+            os.flush()
+            
+            InputStream is = endpoint.getInputStream()
+            String code = DataUtil.readTillRN(is)
+            if (!code.startsWith("200"))
+                throw new IOException("Invalid code $code")
+                
+            // parse all headers
+            Map<String,String> headers = DataUtil.readAllHeaders(is)
+            
+            if (!headers.containsKey("Count"))
+                throw new IOException("No count header")
+            
+            int items = Integer.parseInt(headers['Count'])
+            
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.FETCHING, totalItems: items))
+            
+            JsonSlurper slurper = new JsonSlurper()
+            DataInputStream dis = new DataInputStream(new GZIPInputStream(is))
+            for (int i = 0; i < items; i++) {
+                int size = dis.readUnsignedShort()
+                byte [] tmp = new byte[size]
+                dis.readFully(tmp)
+                def json = slurper.parse(tmp)
+                FeedItem item = FeedItems.objToFeedItem(json, feed.getPublisher())
+                eventBus.publish(new FeedItemFetchedEvent(item: item))
+            }
+            
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.FINISHED))
+        } catch (Exception bad) {
+            log.log(Level.WARNING, "Feed update failed", bad)
+            eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.FAILED)) 
+        } finally {
+            endpoint?.close()
+        }
+    }
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedFetchEvent.groovy

@@ -0,0 +1,10 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.Persona
+
+class FeedFetchEvent extends Event {
+    Persona host
+    FeedFetchStatus status
+    int totalItems
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedItemFetchedEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class FeedItemFetchedEvent extends Event {
+    FeedItem item
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedItemLoadedEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class FeedItemLoadedEvent extends Event {
+    FeedItem item
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedItems.groovy

@@ -0,0 +1,79 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.InfoHash
+import com.muwire.core.Persona
+import com.muwire.core.SharedFile
+import com.muwire.core.files.FileHasher
+import com.muwire.core.util.DataUtil
+
+import net.i2p.data.Base64
+
+class FeedItems {
+    
+    public static def sharedFileToObj(SharedFile sf, int certificates) {
+        def json = [:]
+        json.type = "FeedItem"
+        json.version = 1
+        json.name = Base64.encode(DataUtil.encodei18nString(sf.getFile().getName()))
+        json.infoHash = Base64.encode(sf.getRoot())
+        json.size = sf.getCachedLength()
+        json.pieceSize = sf.getPieceSize()
+        
+        if (sf.getComment() != null)
+            json.comment = sf.getComment()
+        
+        json.certificates = certificates
+        
+        json.timestamp = sf.getPublishedTimestamp()
+        
+        json
+    } 
+    
+    public static FeedItem objToFeedItem(def obj, Persona publisher) throws InvalidFeedItemException {
+        if (obj.timestamp == null)
+            throw new InvalidFeedItemException("No timestamp");
+        if (obj.name == null)
+            throw new InvalidFeedItemException("No name");
+        if (obj.size == null || obj.size <= 0 || obj.size > FileHasher.MAX_SIZE)
+            throw new InvalidFeedItemException("length missing or invalid ${obj.size}")
+        if (obj.pieceSize == null || obj.pieceSize < FileHasher.MIN_PIECE_SIZE_POW2 || obj.pieceSize > FileHasher.MAX_PIECE_SIZE_POW2)
+            throw new InvalidFeedItemException("piece size missing or invalid ${obj.pieceSize}")
+        if (obj.infoHash == null)
+            throw new InvalidFeedItemException("Infohash missing")
+        
+            
+        InfoHash infoHash
+        try {
+            infoHash = new InfoHash(Base64.decode(obj.infoHash))
+        } catch (Exception bad) {
+            throw new InvalidFeedItemException("Invalid infohash", bad)
+        }
+        
+        String name
+        try {
+            name = DataUtil.readi18nString(Base64.decode(obj.name))
+        } catch (Exception bad) {
+            throw new InvalidFeedItemException("Invalid name", bad)
+        }
+        
+        int certificates = 0
+        if (obj.certificates != null)
+            certificates = obj.certificates
+        
+        new FeedItem(publisher, obj.timestamp, name, obj.size, obj.pieceSize, infoHash, certificates, obj.comment)
+    }
+    
+    public static def feedItemToObj(FeedItem item) {
+        def json = [:]
+        json.type = "FeedItem"
+        json.version = 1
+        json.name = Base64.encode(DataUtil.encodei18nString(item.getName()))
+        json.infoHash = Base64.encode(item.getInfoHash().getRoot())
+        json.size = item.getSize()
+        json.pieceSize = item.getPieceSize()
+        json.timestamp = item.getTimestamp()
+        json.certificates = item.getCertificates()
+        json.comment = item.getComment()
+        json
+    }
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedLoadedEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class FeedLoadedEvent extends Event {
+    Feed feed
+}

core/src/main/groovy/com/muwire/core/filefeeds/FeedManager.groovy

@@ -0,0 +1,225 @@
+package com.muwire.core.filefeeds
+
+import java.nio.file.Files
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadFactory
+import java.util.stream.Collectors
+
+import com.muwire.core.EventBus
+import com.muwire.core.Persona
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+import net.i2p.data.Base64
+import net.i2p.util.ConcurrentHashSet
+
+@Log
+class FeedManager {
+    
+    private final EventBus eventBus
+    private final File metadataFolder, itemsFolder
+    private final Map<Persona, Feed> feeds = new ConcurrentHashMap<>()
+    private final Map<Persona, Set<FeedItem>> feedItems = new ConcurrentHashMap<>()
+    
+    private final ExecutorService persister = Executors.newSingleThreadExecutor({r ->
+        new Thread(r, "feed persister")
+    } as ThreadFactory)
+    
+    
+    FeedManager(EventBus eventBus, File home) {
+        this.eventBus = eventBus
+        File feedsFolder = new File(home, "filefeeds")
+        if (!feedsFolder.exists())
+            feedsFolder.mkdir()
+        this.metadataFolder = new File(feedsFolder, "metadata")
+        if (!metadataFolder.exists())
+            metadataFolder.mkdir()
+        this.itemsFolder = new File(feedsFolder, "items")
+        if (!itemsFolder.exists())
+            itemsFolder.mkdir()
+    }
+    
+    public Feed getFeed(Persona persona) {
+        feeds.get(persona)
+    }
+    
+    public Set<FeedItem> getFeedItems(Persona persona) {
+        feedItems.getOrDefault(persona, Collections.emptySet())
+    }
+    
+    public List<Feed> getFeedsToUpdate() {
+        long now = System.currentTimeMillis()
+        feeds.values().stream().
+            filter({Feed f -> !f.getStatus().isActive()}).
+            filter({Feed f -> f.getLastUpdateAttempt() + f.getUpdateInterval() <= now})
+            .collect(Collectors.toList())
+    }
+    
+    void start() {
+        log.info("starting feed manager")
+        persister.submit({loadFeeds()} as Runnable)
+        persister.submit({loadItems()} as Runnable)
+    }
+    
+    void stop() {
+        persister.shutdown()
+    }
+    
+    private void loadFeeds() {
+        def slurper = new JsonSlurper()
+        Files.walk(metadataFolder.toPath()).
+            filter( { it.getFileName().toString().endsWith(".json")}).
+            forEach( {
+                def parsed = slurper.parse(it.toFile())
+                Persona publisher = new Persona(new ByteArrayInputStream(Base64.decode(parsed.publisher)))
+                Feed feed = new Feed(publisher)
+                feed.setUpdateInterval(parsed.updateInterval)
+                feed.setLastUpdated(parsed.lastUpdated)
+                feed.setLastUpdateAttempt(parsed.lastUpdateAttempt)
+                feed.setItemsToKeep(parsed.itemsToKeep)
+                feed.setAutoDownload(parsed.autoDownload)
+                feed.setSequential(parsed.sequential)
+                
+                feed.setStatus(FeedFetchStatus.IDLE)
+                
+                feeds.put(feed.getPublisher(), feed)
+                
+                eventBus.publish(new FeedLoadedEvent(feed : feed))
+            })     
+    }
+    
+    private void loadItems() {
+        def slurper = new JsonSlurper()
+        feeds.keySet().each { persona ->
+            File itemsFile = getItemsFile(feeds[persona])
+            if (!itemsFile.exists())
+                return // no items yet?
+            itemsFile.eachLine { line ->
+                def parsed = slurper.parseText(line)
+                FeedItem item = FeedItems.objToFeedItem(parsed, persona)
+                Set<FeedItem> items = feedItems.get(persona)
+                if (items == null) {
+                    items = new ConcurrentHashSet<>()
+                    feedItems.put(persona, items)
+                }
+                items.add(item)
+                eventBus.publish(new FeedItemLoadedEvent(item : item))
+            }
+        }
+    }
+    
+    void onFeedItemFetchedEvent(FeedItemFetchedEvent e) {
+        Set<FeedItem> set = feedItems.get(e.item.getPublisher())
+        if (set == null) {
+            set = new ConcurrentHashSet<>()
+            feedItems.put(e.getItem().getPublisher(), set)
+        }
+        set.add(e.item)
+    }
+    
+    void onFeedFetchEvent(FeedFetchEvent e) {
+        
+        Feed feed = feeds.get(e.host)
+        if (feed == null) {
+            log.severe("Fetching non-existent feed " + e.host.getHumanReadableName())
+            return
+        }
+        
+        feed.setStatus(e.status)
+        
+        if (e.status.isActive())
+            return
+        
+        if (e.status == FeedFetchStatus.FINISHED) {
+            feed.setStatus(FeedFetchStatus.IDLE)
+            feed.setLastUpdated(e.getTimestamp())
+        }
+        // save feed items, then save feed.  This will save partial fetches too
+        // which is ok because the items are stored in a Set
+        persister.submit({saveFeedItems(e.host)} as Runnable)
+        persister.submit({saveFeedMetadata(feed)} as Runnable)
+    }
+    
+    void onUIFeedConfigurationEvent(UIFeedConfigurationEvent e) {
+        feeds.put(e.feed.getPublisher(), e.feed)
+        persister.submit({saveFeedMetadata(e.feed)} as Runnable)
+    }
+    
+    void onUIFeedDeletedEvent(UIFeedDeletedEvent e) {
+        Feed f = feeds.get(e.host)
+        if (f == null) {
+            log.severe("Deleting a non-existing feed " + e.host.getHumanReadableName())
+            return
+        }
+        persister.submit({deleteFeed(f)} as Runnable)
+    }
+    
+    private void saveFeedItems(Persona publisher) {
+        Set<FeedItem> set = feedItems.get(publisher)
+        if (set == null)
+            return // can happen if nothing was published
+        
+        Feed feed = feeds[publisher]
+        if (feed == null) {
+            log.severe("Persisting items for non-existing feed " + publisher.getHumanReadableName())
+            return
+        }   
+              
+        if (feed.getItemsToKeep() == 0)
+            return
+             
+        List<FeedItem> list = new ArrayList<>(set)
+        if (feed.getItemsToKeep() > 0 && list.size() > feed.getItemsToKeep()) {
+            log.info("will persist ${feed.getItemsToKeep()}/${list.size()} items")
+            list.sort({l, r ->
+                Long.compare(r.getTimestamp(), l.getTimestamp())
+            } as Comparator<FeedItem>)
+            list = list[0..feed.getItemsToKeep() - 1]
+        }
+        
+        
+        File itemsFile = getItemsFile(feed)
+        itemsFile.withPrintWriter { writer -> 
+            list.each { item -> 
+                def obj = FeedItems.feedItemToObj(item)
+                def json = JsonOutput.toJson(obj)
+                writer.println(json)
+            }
+        }
+    }
+    
+    private void saveFeedMetadata(Feed feed) {
+        File metadataFile = getMetadataFile(feed)
+        metadataFile.withPrintWriter { writer ->
+            def json = [:]
+            json.publisher = feed.getPublisher().toBase64()
+            json.itemsToKeep = feed.getItemsToKeep()
+            json.lastUpdated = feed.getLastUpdated()
+            json.updateInterval = feed.getUpdateInterval()
+            json.autoDownload = feed.isAutoDownload()
+            json.sequential = feed.isSequential()
+            json.lastUpdateAttempt = feed.getLastUpdateAttempt()
+            json = JsonOutput.toJson(json)
+            writer.println(json)
+        }
+    }
+    
+    private void deleteFeed(Feed feed) {
+        feeds.remove(feed.getPublisher())
+        feedItems.remove(feed.getPublisher())
+        getItemsFile(feed).delete()
+        getMetadataFile(feed).delete()
+    }
+    
+    private File getItemsFile(Feed feed) {
+        return new File(itemsFolder, feed.getPublisher().destination.toBase32() + ".json")
+    }
+    
+    private File getMetadataFile(Feed feed) {
+        return new File(metadataFolder, feed.getPublisher().destination.toBase32() + ".json")
+    }
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIDownloadFeedItemEvent.groovy

@@ -0,0 +1,9 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+class UIDownloadFeedItemEvent extends Event {
+    FeedItem item
+    File target
+    boolean sequential
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFeedConfigurationEvent.groovy

@@ -0,0 +1,12 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+
+/**
+ * Emitted when configuration of a feed changes.  
+ * The object should already contain the updated values.
+ */
+class UIFeedConfigurationEvent extends Event {
+    Feed feed
+    boolean newFeed
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFeedDeletedEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.Persona
+
+class UIFeedDeletedEvent extends Event {
+    Persona host
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFeedUpdateEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.Persona
+
+class UIFeedUpdateEvent extends Event {
+    Persona host
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFilePublishedEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.SharedFile
+
+class UIFilePublishedEvent extends Event {
+    SharedFile sf
+}

core/src/main/groovy/com/muwire/core/filefeeds/UIFileUnpublishedEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.filefeeds
+
+import com.muwire.core.Event
+import com.muwire.core.SharedFile
+
+class UIFileUnpublishedEvent extends Event {
+    SharedFile sf
+}

core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy

@@ -0,0 +1,173 @@
+package com.muwire.core.files
+
+import com.muwire.core.DownloadedFile
+import com.muwire.core.InfoHash
+import com.muwire.core.Persona
+import com.muwire.core.Service
+import com.muwire.core.SharedFile
+import com.muwire.core.util.DataUtil
+import net.i2p.data.Base64
+import net.i2p.data.Destination
+
+import java.util.stream.Collectors
+
+abstract class BasePersisterService extends Service{
+
+    protected static FileLoadedEvent fromJson(def json) {
+        if (json.file == null || json.length == null || json.infoHash == null || json.hashList == null)
+            throw new IllegalArgumentException()
+        if (!(json.hashList instanceof List))
+            throw new IllegalArgumentException()
+
+        def file = new File(DataUtil.readi18nString(Base64.decode(json.file)))
+        file = file.getCanonicalFile()
+        if (!file.exists() || file.isDirectory())
+            return null
+        long length = Long.valueOf(json.length)
+        if (length != file.length())
+            return null
+
+        List hashList = (List) json.hashList
+        ByteArrayOutputStream baos = new ByteArrayOutputStream()
+        hashList.each {
+            byte [] hash = Base64.decode it.toString()
+            if (hash == null)
+                throw new IllegalArgumentException()
+            baos.write hash
+        }
+        byte[] hashListBytes = baos.toByteArray()
+
+        InfoHash ih = InfoHash.fromHashList(hashListBytes)
+        byte [] root = Base64.decode(json.infoHash.toString())
+        if (root == null)
+            throw new IllegalArgumentException()
+        if (!Arrays.equals(root, ih.getRoot()))
+            return null
+
+        int pieceSize = 0
+        if (json.pieceSize != null)
+            pieceSize = json.pieceSize
+            
+        if (json.sources != null) {
+            List sources = (List)json.sources
+            Set<Destination> sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet()
+            DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet)
+            df.setComment(json.comment)
+            return new FileLoadedEvent(loadedFile : df, infoHash: ih)
+        }
+
+
+        SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize)
+        sf.setComment(json.comment)
+        if (json.downloaders != null)
+            sf.getDownloaders().addAll(json.downloaders)
+        if (json.searchers != null) {
+            json.searchers.each {
+                Persona searcher = null
+                if (it.searcher != null)
+                    searcher = new Persona(new ByteArrayInputStream(Base64.decode(it.searcher)))
+                long timestamp = it.timestamp
+                String query = it.query
+                sf.hit(searcher, timestamp, query)
+            }
+        }
+        return new FileLoadedEvent(loadedFile: sf, infoHash: ih)
+
+    }
+    
+    protected static FileLoadedEvent fromJsonLite(json) {
+        if (json.file == null || json.length == null || json.root == null)
+            throw new IllegalArgumentException()
+            
+        def file = new File(DataUtil.readi18nString(Base64.decode(json.file)))
+        file = file.getCanonicalFile()
+        if (!file.exists() || file.isDirectory())
+            return null
+        long length = Long.valueOf(json.length)
+        if (length != file.length())
+            return null
+            
+        byte[] root = Base64.decode(json.root)
+        InfoHash ih = new InfoHash(root)
+        
+        int pieceSize = 0
+        if (json.pieceSize != null)
+            pieceSize = json.pieceSize
+
+        boolean published = false
+        long publishedTimestamp = -1
+        if (json.published != null && json.published) {
+            published = true
+            publishedTimestamp = json.publishedTimestamp
+        }
+
+        if (json.sources != null) {
+            List sources = (List)json.sources
+            Set<Destination> sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet()
+            DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet)
+            if (published)
+                df.publish(publishedTimestamp)
+            df.setComment(json.comment)
+            return new FileLoadedEvent(loadedFile : df, infoHash: ih)
+        }
+
+
+        SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize)
+        sf.setComment(json.comment)
+        if (published)
+            sf.publish(publishedTimestamp)
+        if (json.downloaders != null)
+            sf.getDownloaders().addAll(json.downloaders)
+        if (json.searchers != null) {
+            json.searchers.each {
+                Persona searcher = null
+                if (it.searcher != null) {
+                    try {
+                        searcher = new Persona(new ByteArrayInputStream(Base64.decode(it.searcher)))
+                    } catch (Exception ignore) {
+                        return 
+                    }
+                }
+                long timestamp = it.timestamp
+                String query = it.query
+                sf.hit(searcher, timestamp, query)
+            }
+        }
+        return new FileLoadedEvent(loadedFile: sf, infoHash: ih)
+    }
+
+    protected static toJson(SharedFile sf) {
+        def json = [:]
+        json.file = sf.getB64EncodedFileName()
+        json.length = sf.getCachedLength()
+        json.root = Base64.encode(sf.getRoot())
+        json.pieceSize = sf.getPieceSize()
+        json.comment = sf.getComment()
+        json.hits = sf.getHits()
+        json.downloaders = sf.getDownloaders()
+
+        if (!sf.searches.isEmpty()) {
+            Set searchers = new HashSet<>()
+            sf.searches.each {
+                def search = [:]
+                if (it.searcher != null)
+                    search.searcher = it.searcher.toBase64()
+                search.timestamp = it.timestamp
+                search.query = it.query
+                searchers.add(search)
+            }
+            json.searchers = searchers
+        }
+
+        if (sf instanceof DownloadedFile) {
+            json.sources = sf.sources.stream().map( {d -> d.toBase64()}).collect(Collectors.toList())
+        }
+        
+        if (sf.isPublished()) {
+            json.published = true
+            json.publishedTimestamp = sf.getPublishedTimestamp()
+        }
+
+        json
+    }
+}

core/src/main/groovy/com/muwire/core/files/DirectoryUnsharedEvent.groovy

@@ -4,8 +4,9 @@ import com.muwire.core.Event
 
 class DirectoryUnsharedEvent extends Event {
     File directory
+    boolean deleted
     
     public String toString() {
-        super.toString() + " unshared directory "+ directory.toString()
+        super.toString() + " unshared directory "+ directory.toString() + " deleted $deleted"
     }
 }

core/src/main/groovy/com/muwire/core/files/DirectoryWatcher.groovy

@@ -15,6 +15,9 @@ import java.util.concurrent.ConcurrentHashMap
 import com.muwire.core.EventBus
 import com.muwire.core.MuWireSettings
 import com.muwire.core.SharedFile
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+import com.muwire.core.files.directories.WatchedDirectoryConvertedEvent
+import com.muwire.core.files.directories.WatchedDirectoryManager
 
 import groovy.util.logging.Log
 import net.i2p.util.SystemVersion
@@ -33,27 +36,27 @@ class DirectoryWatcher {
     }
 
     private final File home
-    private final MuWireSettings muOptions
     private final EventBus eventBus
     private final FileManager fileManager
+    private final WatchedDirectoryManager watchedDirectoryManager
     private final Thread watcherThread, publisherThread
     private final Map<File, Long> waitingFiles = new ConcurrentHashMap<>()
     private final Map<File, WatchKey> watchedDirectories = new ConcurrentHashMap<>()
     private WatchService watchService
     private volatile boolean shutdown
 
-    DirectoryWatcher(EventBus eventBus, FileManager fileManager, File home, MuWireSettings muOptions) {
+    DirectoryWatcher(EventBus eventBus, FileManager fileManager, File home, WatchedDirectoryManager watchedDirectoryManager) {
         this.home = home
-        this.muOptions = muOptions
         this.eventBus = eventBus
         this.fileManager = fileManager
+        this.watchedDirectoryManager = watchedDirectoryManager
         this.watcherThread = new Thread({watch() } as Runnable, "directory-watcher")
         watcherThread.setDaemon(true)
         this.publisherThread = new Thread({publish()} as Runnable, "watched-files-publisher")
         publisherThread.setDaemon(true)
     }
 
-    void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
+    void onWatchedDirectoryConvertedEvent(WatchedDirectoryConvertedEvent e) {
         watchService = FileSystems.getDefault().newWatchService()
         watcherThread.start()
         publisherThread.start()
@@ -71,26 +74,26 @@ class DirectoryWatcher {
         Path path = canonical.toPath()
         WatchKey wk = path.register(watchService, kinds)
         watchedDirectories.put(canonical, wk)
-        
-        if (muOptions.watchedDirectories.add(canonical.toString()))
-            saveMuSettings()
     }
 
     void onDirectoryUnsharedEvent(DirectoryUnsharedEvent e) {
         WatchKey wk = watchedDirectories.remove(e.directory)
         wk?.cancel()
-        
-        if (muOptions.watchedDirectories.remove(e.directory.toString()))
-            saveMuSettings()
     }
     
-    private void saveMuSettings() {
-        File muSettingsFile = new File(home, "MuWire.properties")
-        muSettingsFile.withPrintWriter("UTF-8", {
-            muOptions.write(it)
-        })
+    void onWatchedDirectoryConfigurationEvent(WatchedDirectoryConfigurationEvent e) {
+        if (watchService == null)
+            return // still converting
+        if (!e.autoWatch) {
+            WatchKey wk = watchedDirectories.remove(e.directory)
+            wk?.cancel()
+        } else if (!watchedDirectories.containsKey(e.directory)) {
+            Path path = e.directory.toPath()
+            def wk = path.register(watchService, kinds)
+            watchedDirectories.put(e.directory, wk)
+        } // else it was already watched
     }
-
+    
     private void watch() {
         try {
             while(!shutdown) {
@@ -115,7 +118,7 @@ class DirectoryWatcher {
         File f= join(parent, path)
         log.fine("created entry $f")
         if (f.isDirectory())
-            f.toPath().register(watchService, kinds)
+            eventBus.publish(new FileSharedEvent(file : f, fromWatch : true))
         else
             waitingFiles.put(f, System.currentTimeMillis())
     }
@@ -133,6 +136,10 @@ class DirectoryWatcher {
         SharedFile sf = fileManager.fileToSharedFile.get(f)
         if (sf != null)
             eventBus.publish(new FileUnsharedEvent(unsharedFile : sf, deleted : true))
+        else if (watchedDirectoryManager.isWatched(f)) 
+            eventBus.publish(new DirectoryUnsharedEvent(directory : f, deleted : true))
+        else
+            log.fine("Entry was not relevant");
     }
 
     private static File join(Path parent, Path path) {
@@ -149,7 +156,7 @@ class DirectoryWatcher {
                 waitingFiles.each { file, timestamp ->
                     if (now - timestamp > WAIT_TIME) {
                         log.fine("publishing file $file")
-                        eventBus.publish new FileSharedEvent(file : file)
+                        eventBus.publish new FileSharedEvent(file : file, fromWatch: true)
                         published << file
                     }
                 }

core/src/main/groovy/com/muwire/core/files/FileDownloadedEvent.groovy

@@ -2,6 +2,7 @@ package com.muwire.core.files
 
 import com.muwire.core.DownloadedFile
 import com.muwire.core.Event
+import com.muwire.core.InfoHash
 import com.muwire.core.download.Downloader
 
 import net.i2p.data.Destination
@@ -9,4 +10,5 @@ import net.i2p.data.Destination
 class FileDownloadedEvent extends Event {
     Downloader downloader
     DownloadedFile downloadedFile
+    InfoHash infoHash
 }

core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy

@@ -1,16 +1,18 @@
 package com.muwire.core.files
 
 import com.muwire.core.Event
+import com.muwire.core.InfoHash
 import com.muwire.core.SharedFile
 
 class FileHashedEvent extends Event {
 
     SharedFile sharedFile
+    InfoHash infoHash
     String error
 
     @Override
     public String toString() {
-        super.toString() + " sharedFile " + sharedFile?.file.getAbsolutePath() + " error: $error"
+        super.toString() + " sharedFile " + sharedFile?.file?.getAbsolutePath() + " error: $error"
     }
 
 }

core/src/main/groovy/com/muwire/core/files/FileLoadedEvent.groovy

@@ -1,9 +1,12 @@
 package com.muwire.core.files
 
 import com.muwire.core.Event
+import com.muwire.core.InfoHash
 import com.muwire.core.SharedFile
 
 class FileLoadedEvent extends Event {
 
     SharedFile loadedFile
+    InfoHash infoHash
+    String source
 }

core/src/main/groovy/com/muwire/core/files/FileManager.groovy

@@ -1,5 +1,8 @@
 package com.muwire.core.files
 
+import java.util.stream.Collectors
+import java.util.stream.Stream
+
 import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
@@ -25,6 +28,7 @@ class FileManager {
     final Map<String, Set<File>> commentToFile = new HashMap<>()
     final SearchIndex index = new SearchIndex()
     final FileTree<Void> negativeTree = new FileTree<>()
+    final FileTree<SharedFile> positiveTree = new FileTree<>()
     final Set<File> sideCarFiles = new HashSet<>()
 
     FileManager(EventBus eventBus, MuWireSettings settings) {
@@ -75,7 +79,7 @@ class FileManager {
 
     private void addToIndex(SharedFile sf) {
         log.info("Adding shared file " + sf.getFile())
-        InfoHash infoHash = sf.getInfoHash()
+        InfoHash infoHash = new InfoHash(sf.getRoot())
         Set<SharedFile> existing = rootToFiles.get(infoHash)
         if (existing == null) {
             log.info("adding new root")
@@ -84,6 +88,7 @@ class FileManager {
         }
         existing.add(sf)
         fileToSharedFile.put(sf.file, sf)
+        positiveTree.add(sf.file, sf);
         
         negativeTree.remove(sf.file)
         String parent = sf.getFile().getParent()
@@ -117,7 +122,7 @@ class FileManager {
 
     void onFileUnsharedEvent(FileUnsharedEvent e) {
         SharedFile sf = e.unsharedFile
-        InfoHash infoHash = sf.getInfoHash()
+        InfoHash infoHash = new InfoHash(sf.getRoot())
         Set<SharedFile> existing = rootToFiles.get(infoHash)
         if (existing != null) {
             existing.remove(sf)
@@ -127,6 +132,7 @@ class FileManager {
         }
 
         fileToSharedFile.remove(sf.file)
+        positiveTree.remove(sf.file)
         if (!e.deleted && negativeTree.fileToNode.containsKey(sf.file.getParentFile())) {
             negativeTree.add(sf.file,null)
             saveNegativeTree()
@@ -190,6 +196,10 @@ class FileManager {
     Set<SharedFile> getSharedFiles(byte []root) {
             return rootToFiles.get(new InfoHash(root))
     }
+    
+    boolean isShared(InfoHash infoHash) {
+        rootToFiles.containsKey(infoHash)
+    }
 
     void onSearchEvent(SearchEvent e) {
         // hash takes precedence
@@ -239,14 +249,26 @@ class FileManager {
     void onDirectoryUnsharedEvent(DirectoryUnsharedEvent e) {
         negativeTree.remove(e.directory)
         saveNegativeTree()
-        e.directory.listFiles().each {
-            if (it.isDirectory())
-                eventBus.publish(new DirectoryUnsharedEvent(directory : it))
-            else {
-                SharedFile sf = fileToSharedFile.get(it)
-                if (sf != null)
-                    eventBus.publish(new FileUnsharedEvent(unsharedFile : sf))
+        if (!e.deleted) {
+            e.directory.listFiles().each {
+                if (it.isDirectory())
+                    eventBus.publish(new DirectoryUnsharedEvent(directory : it))
+                else {
+                    SharedFile sf = fileToSharedFile.get(it)
+                    if (sf != null)
+                        eventBus.publish(new FileUnsharedEvent(unsharedFile : sf))
+                }
             }
+        } else {
+             def cb = new DirDeletionCallback()
+             positiveTree.traverse(e.directory, cb)
+             positiveTree.remove(e.directory)
+             cb.unsharedFiles.each { 
+                 eventBus.publish(new FileUnsharedEvent(unsharedFile : it, deleted: true))
+             }
+             cb.subDirs.each {
+                 eventBus.publish(new DirectoryUnsharedEvent(directory : it, deleted : true))
+             }
         }
     }
     
@@ -254,4 +276,34 @@ class FileManager {
         settings.negativeFileTree.clear()
         settings.negativeFileTree.addAll(negativeTree.fileToNode.keySet().collect { it.getAbsolutePath() })
     }
+    
+    public List<SharedFile> getPublishedSince(long timestamp) {
+        synchronized(fileToSharedFile) {
+            fileToSharedFile.values().stream().
+                    filter({sf -> sf.isPublished()}).
+                    filter({sf -> sf.getPublishedTimestamp() >= timestamp}).
+                    collect(Collectors.toList())
+        }
+    }
+    
+    private static class DirDeletionCallback implements FileTreeCallback<SharedFile> {
+        
+        final List<File> subDirs = new ArrayList<>()
+        final List<SharedFile> unsharedFiles = new ArrayList<>()
+
+        @Override
+        public void onDirectoryEnter(File file) {
+            subDirs.add(file)
+        }
+
+        @Override
+        public void onDirectoryLeave() {
+        }
+
+        @Override
+        public void onFile(File file, SharedFile value) {
+            unsharedFiles << value
+        }
+        
+    }
 }

core/src/main/groovy/com/muwire/core/files/FileSharedEvent.groovy

@@ -5,9 +5,10 @@ import com.muwire.core.Event
 class FileSharedEvent extends Event {
 
     File file
+    boolean fromWatch
 
     @Override
     public String toString() {
-        return super.toString() + " file: "+file.getAbsolutePath()
+        return super.toString() + " file: "+file.getAbsolutePath() + " fromWatch: $fromWatch"
     }
 }

core/src/main/groovy/com/muwire/core/files/FileTree.groovy

@@ -23,6 +23,7 @@ class FileTree<T> {
             if (existing == null) {
                 existing = new TreeNode()
                 existing.file = element
+                existing.isFile = element.isFile()
                 existing.parent = current
                 fileToNode.put(element, existing)
                 current.children.add(existing)
@@ -64,7 +65,7 @@ class FileTree<T> {
     private void doTraverse(TreeNode<T> node, FileTreeCallback<T> callback) {
         boolean leave = false
         if (node.file != null) {
-            if (node.file.isFile())
+            if (node.isFile)
                 callback.onFile(node.file, node.value)
             else {
                 leave = true
@@ -88,7 +89,7 @@ class FileTree<T> {
             node = fileToNode.get(parent)
             
         node.children.each { 
-            if (it.file.isFile())
+            if (it.isFile)
                 callback.onFile(it.file, it.value)
             else
                 callback.onDirectory(it.file)
@@ -98,6 +99,7 @@ class FileTree<T> {
     public static class TreeNode<T> {
         TreeNode parent
         File file
+        boolean isFile
         T value;
         final Set<TreeNode> children = new HashSet<>()
         

core/src/main/groovy/com/muwire/core/files/HasherService.groovy

@@ -53,7 +53,6 @@ class HasherService {
 
     private void process(File f) {
         if (f.isDirectory()) {
-            eventBus.publish(new DirectoryWatchedEvent(directory : f))
             f.listFiles().each {
                 eventBus.publish new FileSharedEvent(file: it)
             }
@@ -65,7 +64,8 @@ class HasherService {
             } else {
                 eventBus.publish new FileHashingEvent(hashingFile: f)
                 def hash = hasher.hashFile f
-                eventBus.publish new FileHashedEvent(sharedFile: new SharedFile(f, hash, FileHasher.getPieceSize(f.length())))
+                eventBus.publish new FileHashedEvent(sharedFile: new SharedFile(f, hash.getRoot(), FileHasher.getPieceSize(f.length())),
+                    infoHash : hash)
             }
         }
     }

core/src/main/groovy/com/muwire/core/files/PersisterDoneEvent.groovy

@@ -0,0 +1,12 @@
+package com.muwire.core.files
+
+import com.muwire.core.Event
+
+/**
+ * Should be triggered by the old PersisterService
+ * once it has finished reading the old file
+ *
+ * @see PersisterService
+ */
+class PersisterDoneEvent extends Event{
+}

core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy

@@ -0,0 +1,195 @@
+package com.muwire.core.files
+
+import com.muwire.core.*
+import com.muwire.core.filefeeds.UIFilePublishedEvent
+import com.muwire.core.filefeeds.UIFileUnpublishedEvent
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadFactory
+import java.util.logging.Level
+
+/**
+ * A persister that stores information about the files shared using
+ * individual JSON files in directories.
+ *
+ * The absolute path's 32bit hash to the shared file is used
+ * to build the directory and filename.
+ *
+ * This persister only starts working once the old persister has finished loading
+ * @see PersisterFolderService#getJsonPath
+ */
+@Log
+class PersisterFolderService extends BasePersisterService {
+
+    final static int CUT_LENGTH = 6
+
+    private final Core core;
+    final File location
+    final EventBus listener
+    final int interval
+    final Timer timer
+    final ExecutorService persisterExecutor = Executors.newSingleThreadExecutor({ r ->
+        new Thread(r, "file persister")
+    } as ThreadFactory)
+
+    PersisterFolderService(Core core, File location, EventBus listener) {
+        this.core = core;
+        this.location = location
+        this.listener = listener
+        this.interval = interval
+        timer = new Timer("file-folder persister timer", true)
+    }
+
+    void stop() {
+        timer.cancel()
+        persisterExecutor.shutdown()
+    }
+
+    void onPersisterDoneEvent(PersisterDoneEvent persisterDoneEvent) {
+        log.info("Old persister done")
+        load()
+    }
+
+    void onFileHashedEvent(FileHashedEvent hashedEvent) {
+        if (core.getMuOptions().getAutoPublishSharedFiles() && hashedEvent.sharedFile != null) 
+            hashedEvent.sharedFile.publish(System.currentTimeMillis())
+        persistFile(hashedEvent.sharedFile, hashedEvent.infoHash)
+    }
+
+    void onFileDownloadedEvent(FileDownloadedEvent downloadedEvent) {
+        if (core.getMuOptions().getShareDownloadedFiles()) {
+            if (core.getMuOptions().getAutoPublishSharedFiles())
+                downloadedEvent.downloadedFile.publish(System.currentTimeMillis())
+            persistFile(downloadedEvent.downloadedFile, downloadedEvent.infoHash)
+        }
+    }
+
+    /**
+     * Get rid of the json and hashlists of unshared files
+     * @param unsharedEvent
+     */
+    void onFileUnsharedEvent(FileUnsharedEvent unsharedEvent) {
+        def jsonPath = getJsonPath(unsharedEvent.unsharedFile)
+        def jsonFile = jsonPath.toFile()
+        if(jsonFile.isFile()){
+            jsonFile.delete()
+        }
+        def hashListPath = getHashListPath(unsharedEvent.unsharedFile)
+        def hashListFile = hashListPath.toFile()
+        if (hashListFile.isFile())
+            hashListFile.delete()
+    }
+    
+    void onFileLoadedEvent(FileLoadedEvent loadedEvent) {
+        if(loadedEvent.source == "PersisterService"){
+            log.info("Migrating persisted file from PersisterService: "
+                    + loadedEvent.loadedFile.file.absolutePath.toString())
+            persistFile(loadedEvent.loadedFile, loadedEvent.infoHash)
+        }
+    }
+    
+    void onUICommentEvent(UICommentEvent e) {
+        persistFile(e.sharedFile,null)
+    }
+    
+    void onUIFilePublishedEvent(UIFilePublishedEvent e) {
+        persistFile(e.sf, null)
+    }
+    
+    void onUIFileUnpublishedEvent(UIFileUnpublishedEvent e) {
+        persistFile(e.sf, null)
+    }
+
+    void load() {
+        log.fine("Loading...")
+        Thread.currentThread().setPriority(Thread.MIN_PRIORITY)
+
+        if (location.exists() && location.isDirectory()) {
+            try {
+                _load()
+            }
+            catch (Exception e) {
+                log.log(Level.WARNING, "couldn't load files", e)
+            }
+        } else {
+            location.mkdirs()
+            listener.publish(new AllFilesLoadedEvent())
+        }
+        loaded = true
+    }
+
+    /**
+     * Loads every JSON into memory
+     */
+    private void _load() {
+        int loaded = 0
+        def slurper = new JsonSlurper()
+        Files.walk(location.toPath())
+                .filter({
+                    it.getFileName().toString().endsWith(".json")
+                })
+                .forEach({
+                    def parsed = slurper.parse it.toFile()
+                    def event = fromJsonLite parsed
+                    if (event == null) return
+
+                    log.fine("loaded file $event.loadedFile.file")
+                    listener.publish event
+                    loaded++
+                    if (loaded % 10 == 0)
+                        Thread.sleep(20)
+
+                })
+        listener.publish(new AllFilesLoadedEvent())
+    }
+
+    private void persistFile(SharedFile sf, InfoHash ih) {
+        persisterExecutor.submit({
+            def jsonPath = getJsonPath(sf)
+
+            def startTime = System.currentTimeMillis()
+            jsonPath.parent.toFile().mkdirs()
+            jsonPath.toFile().withPrintWriter { writer ->
+                def json = toJson sf
+                json = JsonOutput.toJson(json)
+                writer.println json
+            }
+            
+            if (ih != null) {
+                def hashListPath = getHashListPath(sf)
+                hashListPath.toFile().bytes = ih.hashList
+            }
+            log.fine("Time(ms) to write json+hashList: " + (System.currentTimeMillis() - startTime))
+        } as Runnable)
+    }
+    private Path getJsonPath(SharedFile sf){
+        def pathHash = sf.getB64PathHash()
+        return Paths.get(
+                location.getAbsolutePath(),
+                pathHash.substring(0, CUT_LENGTH),
+                pathHash.substring(CUT_LENGTH) + ".json"
+        )
+    }
+    
+    private Path getHashListPath(SharedFile sf) {
+        def pathHash = sf.getB64PathHash()
+        return Paths.get(
+                location.getAbsolutePath(),
+                pathHash.substring(0, CUT_LENGTH),
+                pathHash.substring(CUT_LENGTH) + ".hashlist"
+        )
+    }
+    
+    InfoHash loadInfoHash(SharedFile sf) {
+        def path = getHashListPath(sf)
+        InfoHash.fromHashList(path.toFile().bytes)
+    }
+}

core/src/main/groovy/com/muwire/core/files/PersisterService.groovy

@@ -1,40 +1,24 @@
 package com.muwire.core.files
 
-import java.nio.file.CopyOption
-import java.nio.file.Files
-import java.nio.file.StandardCopyOption
+
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 import java.util.concurrent.ThreadFactory
 import java.util.logging.Level
-import java.util.stream.Collectors
 
-import com.muwire.core.DownloadedFile
 import com.muwire.core.EventBus
-import com.muwire.core.InfoHash
-import com.muwire.core.Persona
-import com.muwire.core.Service
-import com.muwire.core.SharedFile
 import com.muwire.core.UILoadedEvent
-import com.muwire.core.util.DataUtil
-
-import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
 import groovy.util.logging.Log
-import net.i2p.data.Base64
-import net.i2p.data.Destination
 
 @Log
-class PersisterService extends Service {
+class PersisterService extends BasePersisterService {
 
     final File location
     final EventBus listener
     final int interval
     final Timer timer
     final FileManager fileManager
-    final ExecutorService persisterExecutor = Executors.newSingleThreadExecutor({ r -> 
-        new Thread(r, "file persister")
-    } as ThreadFactory)
 
     PersisterService(File location, EventBus listener, int interval, FileManager fileManager) {
         this.location = location
@@ -51,10 +35,6 @@ class PersisterService extends Service {
     void onUILoadedEvent(UILoadedEvent e) {
         timer.schedule({load()} as TimerTask, 1)
     }
-    
-    void onUIPersistFilesEvent(UIPersistFilesEvent e) {
-        persistFiles()
-    }
 
     void load() {
         Thread.currentThread().setPriority(Thread.MIN_PRIORITY)
@@ -69,6 +49,7 @@ class PersisterService extends Service {
                         def event = fromJson parsed
                         if (event != null) {
                             log.fine("loaded file $event.loadedFile.file")
+                            event.source = "PersisterService"
                             listener.publish event
                             loaded++
                             if (loaded % 10 == 0)
@@ -76,126 +57,18 @@ class PersisterService extends Service {
                         }
                     }
                 }
-                listener.publish(new AllFilesLoadedEvent())
-            } catch (IllegalArgumentException|NumberFormatException e) {
+                // Backup the old hashes
+                location.renameTo(
+                        new File(location.absolutePath + ".bak")
+                )
+                listener.publish(new PersisterDoneEvent())
+            } catch (Exception e) {
                 log.log(Level.WARNING, "couldn't load files",e)
             }
         } else {
-            listener.publish(new AllFilesLoadedEvent())
+            listener.publish(new PersisterDoneEvent())
         }
-        timer.schedule({persistFiles()} as TimerTask, 1000, interval)
         loaded = true
     }
 
-    private static FileLoadedEvent fromJson(def json) {
-        if (json.file == null || json.length == null || json.infoHash == null || json.hashList == null)
-            throw new IllegalArgumentException()
-        if (!(json.hashList instanceof List))
-            throw new IllegalArgumentException()
-
-        def file = new File(DataUtil.readi18nString(Base64.decode(json.file)))
-        file = file.getCanonicalFile()
-        if (!file.exists() || file.isDirectory())
-            return null
-        long length = Long.valueOf(json.length)
-        if (length != file.length())
-            return null
-
-        List hashList = (List) json.hashList
-        ByteArrayOutputStream baos = new ByteArrayOutputStream()
-        hashList.each {
-            byte [] hash = Base64.decode it.toString()
-            if (hash == null)
-                throw new IllegalArgumentException()
-            baos.write hash
-        }
-        byte[] hashListBytes = baos.toByteArray()
-
-        InfoHash ih = InfoHash.fromHashList(hashListBytes)
-        byte [] root = Base64.decode(json.infoHash.toString())
-        if (root == null)
-            throw new IllegalArgumentException()
-        if (!Arrays.equals(root, ih.getRoot()))
-            return null
-
-        int pieceSize = 0
-        if (json.pieceSize != null)
-            pieceSize = json.pieceSize
-
-        if (json.sources != null) {
-            List sources = (List)json.sources
-            Set<Destination> sourceSet = sources.stream().map({d -> new Destination(d.toString())}).collect Collectors.toSet()
-            DownloadedFile df = new DownloadedFile(file, ih, pieceSize, sourceSet)
-            df.setComment(json.comment)
-            return new FileLoadedEvent(loadedFile : df)
-        }
-
-
-        SharedFile sf = new SharedFile(file, ih, pieceSize)
-        sf.setComment(json.comment)
-        if (json.downloaders != null)
-            sf.getDownloaders().addAll(json.downloaders)
-        if (json.searchers != null) {
-            json.searchers.each {
-                Persona searcher = null
-                if (it.searcher != null)
-                    searcher = new Persona(new ByteArrayInputStream(Base64.decode(it.searcher)))
-                long timestamp = it.timestamp
-                String query = it.query
-                sf.hit(searcher, timestamp, query)
-            }
-        }
-        return new FileLoadedEvent(loadedFile: sf)
-
-    }
-
-    private void persistFiles() {
-        persisterExecutor.submit( {
-            def sharedFiles = fileManager.getSharedFiles()
-
-            File tmp = File.createTempFile("muwire-files", "tmp")
-            tmp.deleteOnExit()
-            tmp.withPrintWriter { writer ->
-                sharedFiles.each { k, v ->
-                    def json = toJson(k,v)
-                    json = JsonOutput.toJson(json)
-                    writer.println json
-                }
-            }
-            Files.copy(tmp.toPath(), location.toPath(), StandardCopyOption.REPLACE_EXISTING)
-            tmp.delete()
-        } as Runnable)
-    }
-
-    private def toJson(File f, SharedFile sf) {
-        def json = [:]
-        json.file = sf.getB64EncodedFileName()
-        json.length = sf.getCachedLength()
-        InfoHash ih = sf.getInfoHash()
-        json.infoHash = sf.getB64EncodedHashRoot()
-        json.pieceSize = sf.getPieceSize()
-        json.hashList = sf.getB64EncodedHashList()
-        json.comment = sf.getComment()
-        json.hits = sf.getHits()
-        json.downloaders = sf.getDownloaders()
-
-        if (!sf.searches.isEmpty()) {
-            Set searchers = new HashSet<>()
-            sf.searches.each {
-                def search = [:]
-                if (it.searcher != null)
-                    search.searcher = it.searcher.toBase64()
-                search.timestamp = it.timestamp
-                search.query = it.query
-                searchers.add(search)
-            }
-            json.searchers = searchers
-        }
-        
-        if (sf instanceof DownloadedFile) {
-            json.sources = sf.sources.stream().map( {d -> d.toBase64()}).collect(Collectors.toList())
-        }
-
-        json
-    }
 }

core/src/main/groovy/com/muwire/core/files/UIPersistFilesEvent.groovy

@@ -1,6 +0,0 @@
-package com.muwire.core.files
-
-import com.muwire.core.Event
-
-class UIPersistFilesEvent extends Event {
-}

core/src/main/groovy/com/muwire/core/files/directories/UISyncDirectoryEvent.groovy

@@ -0,0 +1,7 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+class UISyncDirectoryEvent extends Event {
+    File directory
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectory.groovy

@@ -0,0 +1,37 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.util.DataUtil
+
+import net.i2p.data.Base64
+
+class WatchedDirectory {
+    final File directory
+    final String encodedName
+    boolean autoWatch
+    int syncInterval
+    long lastSync
+    
+    WatchedDirectory(File directory) {
+        this.directory = directory.getCanonicalFile()
+        this.encodedName = Base64.encode(DataUtil.encodei18nString(directory.getAbsolutePath()))
+    }
+    
+    def toJson() {
+        def rv = [:]
+        rv.directory = encodedName
+        rv.autoWatch = autoWatch
+        rv.syncInterval = syncInterval
+        rv.lastSync = lastSync
+        rv
+    }
+    
+    static WatchedDirectory fromJson(def json) {
+        String dirName = DataUtil.readi18nString(Base64.decode(json.directory))
+        File dir = new File(dirName)
+        def rv = new WatchedDirectory(dir)
+        rv.autoWatch = json.autoWatch
+        rv.syncInterval = json.syncInterval
+        rv.lastSync = json.lastSync
+        rv
+    }
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryConfigurationEvent.groovy

@@ -0,0 +1,9 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+class WatchedDirectoryConfigurationEvent extends Event {
+    File directory
+    boolean autoWatch
+    int syncInterval
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryConvertedEvent.groovy

@@ -0,0 +1,10 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+/**
+ * Emitted when converting an old watched directory entry to the
+ * new format.
+ */
+class WatchedDirectoryConvertedEvent extends Event {
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryConverter.groovy

@@ -0,0 +1,27 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Core
+import com.muwire.core.files.AllFilesLoadedEvent
+
+/**
+ * converts the setting-based format to new folder-based format.
+ */
+class WatchedDirectoryConverter {
+    
+    private final Core core
+    
+    WatchedDirectoryConverter(Core core) {
+        this.core = core
+    }
+    
+    void onAllFilesLoadedEvent(AllFilesLoadedEvent e) {
+        core.getMuOptions().getWatchedDirectories().each {
+            File directory = new File(it)
+            directory = directory.getCanonicalFile() 
+            core.eventBus.publish(new WatchedDirectoryConfigurationEvent(directory : directory, autoWatch: true))
+        }
+        core.getMuOptions().getWatchedDirectories().clear()
+        core.saveMuSettings()
+        core.eventBus.publish(new WatchedDirectoryConvertedEvent())
+    }
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectoryManager.groovy

@@ -0,0 +1,220 @@
+package com.muwire.core.files.directories
+
+import java.nio.file.Files
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadFactory
+import java.util.stream.Stream
+
+import com.muwire.core.EventBus
+import com.muwire.core.SharedFile
+import com.muwire.core.files.DirectoryUnsharedEvent
+import com.muwire.core.files.DirectoryWatchedEvent
+import com.muwire.core.files.FileListCallback
+import com.muwire.core.files.FileManager
+import com.muwire.core.files.FileSharedEvent
+import com.muwire.core.files.FileUnsharedEvent
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+
+@Log
+class WatchedDirectoryManager {
+    
+    private final File home
+    private final EventBus eventBus
+    private final FileManager fileManager
+    
+    private final Map<File, WatchedDirectory> watchedDirs = new ConcurrentHashMap<>()
+    
+    private final ExecutorService diskIO = Executors.newSingleThreadExecutor({r -> 
+        Thread t = new Thread(r, "disk-io")
+        t.setDaemon(true)
+        t
+    } as ThreadFactory)
+    
+    private final Timer timer = new Timer("directory-timer", true)
+    
+    private boolean converting = true
+    
+    WatchedDirectoryManager(File home, EventBus eventBus, FileManager fileManager) {
+        this.home = new File(home, "directories")
+        this.home.mkdir()
+        this.eventBus = eventBus
+        this.fileManager = fileManager
+    }
+    
+    public boolean isWatched(File f) {
+        watchedDirs.containsKey(f)
+    }
+    
+    public Stream<WatchedDirectory> getWatchedDirsStream() {
+        watchedDirs.values().stream()
+    }
+    
+    public void shutdown() {
+        diskIO.shutdown()
+        timer.cancel()
+    }
+    
+    void onUISyncDirectoryEvent(UISyncDirectoryEvent e) {
+        def wd = watchedDirs.get(e.directory)
+        if (wd == null) {
+            log.warning("Got a sync event for non-watched dir ${e.directory}")
+            return
+        }
+        diskIO.submit({sync(wd, System.currentTimeMillis())} as Runnable)
+    }
+    
+    void onWatchedDirectoryConfigurationEvent(WatchedDirectoryConfigurationEvent e) {
+        if (converting) {
+            def newDir = new WatchedDirectory(e.directory)
+            // conversion is always autowatch really
+            newDir.autoWatch = e.autoWatch
+            persist(newDir)
+        } else {
+            def wd = watchedDirs.get(e.directory)
+            if (wd == null) {
+                log.severe("got a configuration event for a non-watched directory ${e.directory}")
+                return
+            }
+            wd.autoWatch = e.autoWatch
+            wd.syncInterval = e.syncInterval
+            persist(wd)
+        }
+    }
+    
+    void onWatchedDirectoryConvertedEvent(WatchedDirectoryConvertedEvent e) {
+        converting = false
+        diskIO.submit({
+            def slurper = new JsonSlurper()
+            Files.walk(home.toPath()).filter({
+                it.getFileName().toString().endsWith(".json")
+            }).
+            forEach {
+                def parsed = slurper.parse(it.toFile())
+                WatchedDirectory wd = WatchedDirectory.fromJson(parsed)
+                watchedDirs.put(wd.directory, wd)
+            }
+            watchedDirs.values().stream().filter({it.autoWatch}).forEach {
+                eventBus.publish(new DirectoryWatchedEvent(directory : it.directory))
+                eventBus.publish(new FileSharedEvent(file : it.directory))
+            }
+            timer.schedule({sync()} as TimerTask, 1000, 1000)
+        } as Runnable)
+    }
+    
+    private void persist(WatchedDirectory dir) {
+        diskIO.submit({doPersist(dir)} as Runnable)
+    }
+    
+    private void doPersist(WatchedDirectory dir) {
+        def json = JsonOutput.toJson(dir.toJson())
+        def targetFile = new File(home, dir.getEncodedName() + ".json")
+        targetFile.text = json
+    }
+    
+    void onFileSharedEvent(FileSharedEvent e) {
+        if (e.file.isFile() || watchedDirs.containsKey(e.file))
+            return
+        
+        def wd = new WatchedDirectory(e.file)
+        if (e.fromWatch) {
+            // parent should be already watched, copy settings
+            def parent = watchedDirs.get(e.file.getParentFile())
+            if (parent == null) {
+                log.severe("watching found a directory without a watched parent? ${e.file}")
+                return
+            }
+            wd.autoWatch = parent.autoWatch
+            wd.syncInterval = parent.syncInterval
+        } else
+            wd.autoWatch = true
+        
+        watchedDirs.put(wd.directory, wd)
+        persist(wd)
+        if (wd.autoWatch)
+            eventBus.publish(new DirectoryWatchedEvent(directory: wd.directory))
+    }
+    
+    void onDirectoryUnsharedEvent(DirectoryUnsharedEvent e) {
+        def wd = watchedDirs.remove(e.directory)
+        if (wd == null) {
+            log.warning("unshared a directory that wasn't watched? ${e.directory}")
+            return
+        }
+        
+        File persistFile = new File(home, wd.getEncodedName() + ".json")
+        persistFile.delete()
+    }
+    
+    private void sync() {
+        long now = System.currentTimeMillis()
+        watchedDirs.values().stream().
+            filter({!it.autoWatch}).
+            filter({it.syncInterval > 0}).
+            filter({it.lastSync + it.syncInterval * 1000 < now}).
+            forEach({wd -> diskIO.submit({sync(wd, now)} as Runnable )})
+    }
+    
+    private void sync(WatchedDirectory wd, long now) {
+        log.fine("syncing ${wd.directory}")
+        wd.lastSync = now
+        doPersist(wd)
+        eventBus.publish(new WatchedDirectorySyncEvent(directory: wd.directory, when: now))
+        
+        def cb = new DirSyncCallback()
+        fileManager.positiveTree.list(wd.directory, cb)
+        
+        Set<File> filesOnFS = new HashSet<>()
+        Set<File> dirsOnFS = new HashSet<>()
+        wd.directory.listFiles().each {
+            File canonical = it.getCanonicalFile() 
+            if (canonical.isFile())
+                filesOnFS.add(canonical)
+            else
+                dirsOnFS.add(canonical)
+        }
+        
+        Set<File> addedFiles = new HashSet<>(filesOnFS)
+        addedFiles.removeAll(cb.files)
+        addedFiles.each { 
+            eventBus.publish(new FileSharedEvent(file : it, fromWatch : true))
+        }
+        Set<File> addedDirs = new HashSet<>(dirsOnFS)
+        addedDirs.removeAll(cb.dirs)
+        addedDirs.each { 
+            eventBus.publish(new FileSharedEvent(file : it, fromWatch : true))
+        }
+        
+        Set<File> deletedFiles = new HashSet<>(cb.files)
+        deletedFiles.removeAll(filesOnFS)
+        deletedFiles.each {
+            eventBus.publish(new FileUnsharedEvent(unsharedFile : fileManager.getFileToSharedFile().get(it), deleted : true))
+        }
+        Set<File> deletedDirs = new HashSet<>(cb.dirs)
+        deletedDirs.removeAll(dirsOnFS)
+        deletedDirs.each {
+            eventBus.publish(new DirectoryUnsharedEvent(directory : it, deleted: true))
+        }
+    }
+    
+    private static class DirSyncCallback implements FileListCallback<SharedFile> {
+        
+        private final Set<File> files = new HashSet<>()
+        private final Set<File> dirs = new HashSet<>()
+
+        @Override
+        public void onFile(File f, SharedFile value) {
+            files.add(f)
+        }
+
+        @Override
+        public void onDirectory(File f) {
+            dirs.add(f)
+        }
+        
+    }
+}

core/src/main/groovy/com/muwire/core/files/directories/WatchedDirectorySyncEvent.groovy

@@ -0,0 +1,8 @@
+package com.muwire.core.files.directories
+
+import com.muwire.core.Event
+
+class WatchedDirectorySyncEvent extends Event {
+    File directory
+    long when
+}

core/src/main/groovy/com/muwire/core/hostcache/CacheClient.groovy

@@ -1,5 +1,7 @@
 package com.muwire.core.hostcache
 
+import java.util.concurrent.atomic.AtomicBoolean
+
 import com.muwire.core.EventBus
 import com.muwire.core.MuWireSettings
 import com.muwire.core.connection.ConnectionManager
@@ -27,6 +29,7 @@ class CacheClient {
     final long interval
     final MuWireSettings settings
     final Timer timer
+    private final AtomicBoolean stopped = new AtomicBoolean();
 
     public CacheClient(EventBus eventBus, HostCache cache,
         ConnectionManager manager, I2PSession session,
@@ -47,9 +50,12 @@ class CacheClient {
 
     void stop() {
         timer.cancel()
+        stopped.set(true)
     }
 
     private void queryIfNeeded() {
+        if (stopped.get())
+            return
         if (!manager.getConnections().isEmpty())
             return
         if (!cache.getHosts(1).isEmpty())
@@ -65,7 +71,12 @@ class CacheClient {
         options.setSendLeaseSet(true)
         CacheServers.getCacheServers().each {
             log.info "Querying hostcache ${it.toBase32()}"
-            session.sendMessage(it, ping, 0, ping.length, I2PSession.PROTO_DATAGRAM, 1, 0, options)
+            try {
+                session.sendMessage(it, ping, 0, ping.length, I2PSession.PROTO_DATAGRAM, 1, 0, options)
+            } catch (Exception e) {
+                if (!stopped.get())
+                    throw e
+            }
         }
     }
 

core/src/main/groovy/com/muwire/core/hostcache/CacheServers.groovy

@@ -8,10 +8,8 @@ class CacheServers {
     private static Set<Destination> CACHES = [
             // zlatinb
             new Destination("Wddh2E6FyyXBF7SvUYHKdN-vjf3~N6uqQWNeBDTM0P33YjiQCOsyedrjmDZmWFrXUJfJLWnCb5bnKezfk4uDaMyj~uvDG~yvLVcFgcPWSUd7BfGgym-zqcG1q1DcM8vfun-US7YamBlmtC6MZ2j-~Igqzmgshita8aLPCfNAA6S6e2UMjjtG7QIXlxpMec75dkHdJlVWbzrk9z8Qgru3YIk0UztYgEwDNBbm9wInsbHhr3HtAfa02QcgRVqRN2PnQXuqUJs7R7~09FZPEviiIcUpkY3FeyLlX1sgQFBeGeA96blaPvZNGd6KnNdgfLgMebx5SSxC-N4KZMSMBz5cgonQF3~m2HHFRSI85zqZNG5X9bJN85t80ltiv1W1es8ZnQW4es11r7MrvJNXz5bmSH641yJIvS6qI8OJJNpFVBIQSXLD-96TayrLQPaYw~uNZ-eXaE6G5dYhiuN8xHsFI1QkdaUaVZnvDGfsRbpS5GtpUbBDbyLkdPurG0i7dN1wAAAA"),
-            // sNL
-            new Destination("JC63wJNOqSJmymkj4~UJWywBTvDGikKMoYP0HX2Wz9c5l3otXSkwnxWAFL4cKr~Ygh3BNNi2t93vuLIiI1W8AsE42kR~PwRx~Y-WvIHXR6KUejRmOp-n8WidtjKg9k4aDy428uSOedqXDxys5mpoeQXwDsv1CoPTTwnmb1GWFy~oTGIsCguCl~aJWGnqiKarPO3GJQ~ev-NbvAQzUfC3HeP1e6pdI5CGGjExahTCID5UjpJw8GaDXWlGmYWWH303Xu4x-vAHQy1dJLsOBCn8dZravsn5BKJk~j0POUon45CCx-~NYtaPe0Itt9cMdD2ciC76Rep1D0X0sm1SjlSs8sZ52KmF3oaLZ6OzgI9QLMIyBUrfi41sK5I0qTuUVBAkvW1xr~L-20dYJ9TrbOaOb2-vDIfKaxVi6xQOuhgQDiSBhd3qv2m0xGu-BM9DQYfNA0FdMjnZmqjmji9RMavzQSsVFIbQGLbrLepiEFlb7TseCK5UtRp8TxnG7L4gbYevBQAEAAcAAA=="),
-            // dark_trion
-            new Destination("Gec9L29FVcQvYDgpcYuEYdltJn06PPoOWAcAM8Af-gDm~ehlrJcwlLXXs0hidq~yP2A0X7QcDi6i6shAfuEofTchxGJl8LRNqj9lio7WnB7cIixXWL~uCkD7Np5LMX0~akNX34oOb9RcBYVT2U5rFGJmJ7OtBv~IBkGeLhsMrqaCjahd0jdBO~QJ-t82ZKZhh044d24~JEfF9zSJxdBoCdAcXzryGNy7sYtFVDFsPKJudAxSW-UsSQiGw2~k-TxyF0r-iAt1IdzfNu8Lu0WPqLdhDYJWcPldx2PR5uJorI~zo~z3I5RX3NwzarlbD4nEP5s65ahPSfVCEkzmaJUBgP8DvBqlFaX89K4nGRYc7jkEjJ8cX4L6YPXUpTPWcfKkW259WdQY3YFh6x7rzijrGZewpczOLCrt-bZRYgDrUibmZxKZmNhy~lQu4gYVVjkz1i4tL~DWlhIc4y0x2vItwkYLArPPi~ejTnt-~Lhb7oPMXRcWa3UrwGKpFvGZY4NXBQAEAAcAAA==")
+            // echelon
+            new Destination("2MJTl8gYVPK43iJZJa~-5K1OchgPaPHXpqZmKIiKFvxyy8BlIJzUSrF4mazdta--shFHISfT0PEeI95j1yDyKMpGxatUyjSt3ZnyTfAehQR-H2kYV9FvjHo68uA9X5AaGYHKRYLuWMkihMXygd8ywoLjZtFP0UbKMPggfOZaWmjHF4081XoUXt~7MEAeYSQowndiUx0AH3HxNEiv0N373JJS61OsIXb5ctqVKkwIiX1R0ZxESzpP9Xwp8-T0ou8fsLksygbKyH~3K1CyTHjTS51Ux-U-CjOPH9rtCOjjAaifdyMpK0PxW1fVdoGswFywTz9Q-6DUMsIu5TsPMF0-UO1Wn8vCpVAWbBJAOtKCfBrGzp-E~GCbfCNs5xY19nLobMD5ehjsBdI1lXwGDCQ7kBOwC58uuC3BOoazgrB6IrGskyMTexawtthO9mhuPm91bq4xhNaCYHAe059xg5emnM7jFBVzQgjaZ5lOLn~HqcWofJ7oc0doE6XI6kOo~YncBQAEAAcAAA==")
         ]
 
     static List<Destination> getCacheServers() {

core/src/main/groovy/com/muwire/core/mesh/Mesh.groovy

@@ -10,7 +10,7 @@ import net.i2p.util.ConcurrentHashSet
 class Mesh {
     private final InfoHash infoHash
     private final Set<Persona> sources = new ConcurrentHashSet<>()
-    private final Pieces pieces
+    final Pieces pieces
 
     Mesh(InfoHash infoHash, Pieces pieces) {
         this.infoHash = infoHash

core/src/main/groovy/com/muwire/core/search/ResultsSender.groovy

@@ -77,18 +77,19 @@ class ResultsSender {
                 if (it.getComment() != null) {
                     comment = DataUtil.readi18nString(Base64.decode(it.getComment()))
                 }
-                int certificates = certificateManager.getByInfoHash(it.getInfoHash()).size()
+                int certificates = certificateManager.getByInfoHash(new InfoHash(it.getRoot())).size()
                 def uiResultEvent = new UIResultEvent( sender : me,
                     name : it.getFile().getName(),
                     size : length,
-                    infohash : it.getInfoHash(),
+                    infohash : new InfoHash(it.getRoot()),
                     pieceSize : pieceSize,
                     uuid : uuid,
                     browse : settings.browseFiles,
                     sources : suggested,
                     comment : comment,
                     certificates : certificates,
-                    chat : chatServer.running.get() && settings.advertiseChat
+                    chat : chatServer.running.get() && settings.advertiseChat,
+                    feed : settings.fileFeed && settings.advertiseFeed
                     )
                 uiResultEvents << uiResultEvent
             }
@@ -119,7 +120,7 @@ class ResultsSender {
                         me.write(os)
                         os.writeShort((short)results.length)
                         results.each {
-                            int certificates = certificateManager.getByInfoHash(it.getInfoHash()).size()
+                            int certificates = certificateManager.getByInfoHash(new InfoHash(it.getRoot())).size()
                             def obj = sharedFileToObj(it, settings.browseFiles, certificates)
                             def json = jsonOutput.toJson(obj)
                             os.writeShort((short)json.length())
@@ -138,10 +139,12 @@ class ResultsSender {
                         os.write("Count: $results.length\r\n".getBytes(StandardCharsets.US_ASCII))
                         boolean chat = chatServer.running.get() && settings.advertiseChat
                         os.write("Chat: $chat\r\n".getBytes(StandardCharsets.US_ASCII))
+                        boolean feed = settings.fileFeed && settings.advertiseFeed
+                        os.write("Feed: $feed\r\n".getBytes(StandardCharsets.US_ASCII))
                         os.write("\r\n".getBytes(StandardCharsets.US_ASCII))
                         DataOutputStream dos = new DataOutputStream(new GZIPOutputStream(os))
                         results.each { 
-                            int certificates = certificateManager.getByInfoHash(it.getInfoHash()).size()
+                            int certificates = certificateManager.getByInfoHash(new InfoHash(it.getRoot())).size()
                             def obj = sharedFileToObj(it, settings.browseFiles, certificates)
                             def json = jsonOutput.toJson(obj)
                             dos.writeShort((short)json.length())
@@ -170,7 +173,7 @@ class ResultsSender {
         obj.type = "Result"
         obj.version = 2
         obj.name = encodedName
-        obj.infohash = Base64.encode(sf.getInfoHash().getRoot())
+        obj.infohash = Base64.encode(sf.getRoot())
         obj.size = sf.getCachedLength()
         obj.pieceSize = sf.getPieceSize()
 

core/src/main/groovy/com/muwire/core/search/UIResultEvent.groovy

@@ -18,7 +18,8 @@ class UIResultEvent extends Event {
     boolean browse
     int certificates
     boolean chat
-
+    boolean feed
+    
     @Override
     public String toString() {
         super.toString() + "name:$name size:$size sender:${sender.getHumanReadableName()} pieceSize $pieceSize"

core/src/main/groovy/com/muwire/core/tracker/TrackerResponder.groovy

@@ -0,0 +1,214 @@
+package com.muwire.core.tracker
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.logging.Level
+import java.util.stream.Collectors
+
+import com.muwire.core.Constants
+import com.muwire.core.InfoHash
+import com.muwire.core.MuWireSettings
+import com.muwire.core.Persona
+import com.muwire.core.download.DownloadManager
+import com.muwire.core.download.Pieces
+import com.muwire.core.files.FileManager
+import com.muwire.core.mesh.Mesh
+import com.muwire.core.mesh.MeshManager
+import com.muwire.core.trust.TrustLevel
+import com.muwire.core.trust.TrustService
+import com.muwire.core.util.DataUtil
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import groovy.util.logging.Log
+import net.i2p.client.I2PSession
+import net.i2p.client.I2PSessionMuxedListener
+import net.i2p.client.SendMessageOptions
+import net.i2p.client.datagram.I2PDatagramDissector
+import net.i2p.client.datagram.I2PDatagramMaker
+import net.i2p.data.Base64
+
+@Log
+class TrackerResponder {
+    private final I2PSession i2pSession
+    private final MuWireSettings muSettings
+    private final FileManager fileManager
+    private final DownloadManager downloadManager
+    private final MeshManager meshManager
+    private final TrustService trustService
+    private final Persona me
+    
+    private final Map<UUID,Long> uuids = new HashMap<>()
+    private final Timer expireTimer = new Timer("tracker-responder-timer", true)
+    
+    private static final long UUID_LIFETIME = 10 * 60 * 1000
+    
+    TrackerResponder(I2PSession i2pSession, MuWireSettings muSettings,
+        FileManager fileManager, DownloadManager downloadManager,
+        MeshManager meshManager, TrustService trustService,
+        Persona me) {
+        this.i2pSession = i2pSession
+        this.muSettings = muSettings
+        this.fileManager = fileManager
+        this.downloadManager = downloadManager
+        this.meshManager = meshManager
+        this.trustService = trustService
+        this.me = me
+    }
+    
+    void start() {
+        i2pSession.addMuxedSessionListener(new Listener(), I2PSession.PROTO_DATAGRAM, Constants.TRACKER_PORT)
+        expireTimer.schedule({expireUUIDs()} as TimerTask, UUID_LIFETIME, UUID_LIFETIME)
+    }
+    
+    void stop() {
+        expireTimer.cancel()
+    }
+    
+    private void expireUUIDs() {
+        final long now = System.currentTimeMillis()
+        synchronized(uuids) {
+            for (Iterator<UUID> iter = uuids.keySet().iterator(); iter.hasNext();) {
+                UUID uuid = iter.next();
+                Long time = uuids.get(uuid)
+                if (now - time > UUID_LIFETIME)
+                    iter.remove()
+            }
+        }
+    }
+    
+    private void respond(host, json) {
+        log.info("responding to host $host with json $json")
+        
+        def message = JsonOutput.toJson(json)
+        def maker = new I2PDatagramMaker(i2pSession)
+        message = maker.makeI2PDatagram(message.bytes)
+        def options = new SendMessageOptions()
+        options.setSendLeaseSet(false)
+        i2pSession.sendMessage(host, message, 0, message.length, I2PSession.PROTO_DATAGRAM, Constants.TRACKER_PORT, Constants.TRACKER_PORT, options)
+    }
+    
+    class Listener implements I2PSessionMuxedListener {
+
+        @Override
+        public void messageAvailable(I2PSession session, int msgId, long size) {
+        }
+
+        @Override
+        public void messageAvailable(I2PSession session, int msgId, long size, int proto, int fromport, int toport) {
+            if (proto != I2PSession.PROTO_DATAGRAM) {
+                log.warning "Received unexpected protocol $proto"
+                return
+            }
+            
+            byte[] payload = session.receiveMessage(msgId)
+            def dissector = new I2PDatagramDissector()
+            try {
+                dissector.loadI2PDatagram(payload)
+                def sender = dissector.getSender()
+                
+                log.info("got a tracker datagram from ${sender.toBase32()}")
+                
+                // if not trusted, just drop it
+                TrustLevel trustLevel = trustService.getLevel(sender)
+                
+                if (trustLevel == TrustLevel.DISTRUSTED || 
+                    (trustLevel == TrustLevel.NEUTRAL && !muSettings.allowUntrusted)) {
+                    log.info("dropping, untrusted")
+                    return
+                }
+                
+                payload = dissector.getPayload()
+                def slurper = new JsonSlurper()
+                def json = slurper.parse(payload)
+                
+                if (json.type != "TrackerPing") {
+                    log.warning("unknown type $json.type")
+                    return
+                }
+                
+                def response = [:]
+                response.type = "TrackerPong"
+                response.me = me.toBase64()
+
+                if (json.infoHash == null) {
+                    log.warning("infoHash missing")
+                    return
+                }
+                
+                if (json.uuid == null) {
+                    log.warning("uuid missing")
+                    return
+                }
+                
+                UUID uuid = UUID.fromString(json.uuid)
+                synchronized(uuids) {
+                    if (uuids.containsKey(uuid)) {
+                        log.warning("duplicate uuid $uuid")
+                        return
+                    }
+                    uuids.put(uuid, System.currentTimeMillis())
+                }
+                response.uuid = json.uuid
+                                
+                if (!muSettings.allowTracking) {
+                    response.code = 403
+                    respond(sender, response)
+                    return
+                }
+                
+                if (json.version != 1) {
+                    log.warning("unknown version $json.version")
+                    response.code = 400
+                    response.message = "I only support version 1"
+                    respond(sender,response)
+                    return
+                }
+                
+                byte[] infoHashBytes = Base64.decode(json.infoHash)
+                InfoHash infoHash = new InfoHash(infoHashBytes)
+                
+                log.info("servicing request for infoHash ${json.infoHash} with uuid ${json.uuid}")
+                
+                if (!(fileManager.isShared(infoHash) || downloadManager.isDownloading(infoHash))) {
+                    response.code = 404
+                    respond(sender, response)
+                    return
+                }
+
+                Mesh mesh = meshManager.get(infoHash)   
+                  
+                if (fileManager.isShared(infoHash))
+                    response.code = 200
+                else if (mesh != null) {
+                    response.code = 206
+                    Pieces pieces = mesh.getPieces()
+                    response.xHave = DataUtil.encodeXHave(pieces, pieces.getnPieces())
+                }
+                
+                if (mesh != null)
+                    response.altlocs = mesh.getRandom(10, me).stream().map({it.toBase64()}).collect(Collectors.toList())
+                    
+                respond(sender,response)
+            } catch (Exception e) {
+                log.log(Level.WARNING, "invalid datagram", e)
+            }
+        }
+
+        @Override
+        public void reportAbuse(I2PSession session, int severity) {
+        }
+
+        @Override
+        public void disconnected(I2PSession session) {
+            log.severe("session disconnected")
+        }
+
+        @Override
+        public void errorOccurred(I2PSession session, String message, Throwable error) {
+            log.log(Level.SEVERE, message, error)
+        }
+        
+    }
+    
+    
+}

core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy

@@ -2,6 +2,7 @@ package com.muwire.core.update
 
 import java.util.logging.Level
 
+import com.muwire.core.Constants
 import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 import com.muwire.core.MuWireSettings
@@ -63,7 +64,7 @@ class UpdateClient {
     }
 
     void start() {
-        session.addMuxedSessionListener(new Listener(), I2PSession.PROTO_DATAGRAM, 2)
+        session.addMuxedSessionListener(new Listener(), I2PSession.PROTO_DATAGRAM, Constants.UPDATE_PORT)
         timer.schedule({checkUpdate()} as TimerTask, 60000, 60 * 60 * 1000)
     }
 
@@ -83,7 +84,7 @@ class UpdateClient {
     }
 
     void onFileDownloadedEvent(FileDownloadedEvent e) {
-        if (e.downloadedFile.infoHash != updateInfoHash)
+        if (e.infoHash != updateInfoHash)
             return
         updateDownloading = false
         eventBus.publish(new UpdateDownloadedEvent(version : version, signer : signer, text : text))
@@ -108,7 +109,7 @@ class UpdateClient {
         ping = maker.makeI2PDatagram(ping.bytes)
         def options = new SendMessageOptions()
         options.setSendLeaseSet(true)
-        session.sendMessage(UpdateServers.UPDATE_SERVER, ping, 0, ping.length, I2PSession.PROTO_DATAGRAM, 2, 0, options)
+        session.sendMessage(UpdateServers.UPDATE_SERVER, ping, 0, ping.length, I2PSession.PROTO_DATAGRAM, Constants.UPDATE_PORT, 0, options)
     }
 
     class Listener implements I2PSessionMuxedListener {

core/src/main/groovy/com/muwire/core/upload/ContentUploader.groovy

@@ -106,7 +106,7 @@ class ContentUploader extends Uploader {
             return done ? 100 : 0
         int position = mapped.position()
         int total = request.getRange().end - request.getRange().start
-        (int)(position * 100.0 / total)
+        (int)(position * 100.0d / total)
     }
 
     @Override

core/src/main/groovy/com/muwire/core/upload/HashListUploader.groovy

@@ -45,7 +45,7 @@ class HashListUploader extends Uploader {
 
     @Override
     public synchronized int getProgress() {
-        (int)(mapped.position() * 100.0 / mapped.capacity())
+        (int)(mapped.position() * 100.0d / mapped.capacity())
     }
 
     @Override

core/src/main/groovy/com/muwire/core/upload/Request.groovy

@@ -22,7 +22,7 @@ class Request {
 
     static Request parseContentRequest(InfoHash infoHash, InputStream is) throws IOException {
 
-        Map<String, String> headers = parseHeaders(is)
+        Map<String, String> headers = DataUtil.readAllHeaders(is)
 
         if (!headers.containsKey("Range"))
             throw new IOException("Range header not found")
@@ -60,7 +60,7 @@ class Request {
     }
 
     static Request parseHashListRequest(InfoHash infoHash, InputStream is) throws IOException {
-        Map<String,String> headers = parseHeaders(is)
+        Map<String,String> headers = DataUtil.readAllHeaders(is)
         Persona downloader = null
         if (headers.containsKey("X-Persona")) {
             def encoded = headers["X-Persona"].trim()
@@ -69,55 +69,4 @@ class Request {
         }
         new HashListRequest(infoHash : infoHash, headers : headers, downloader : downloader)
     }
-
-    private static Map<String, String> parseHeaders(InputStream is) {
-        Map<String,String> headers = new HashMap<>()
-        byte [] tmp = new byte[Constants.MAX_HEADER_SIZE]
-        while(headers.size() < Constants.MAX_HEADERS) {
-            boolean r = false
-            boolean n = false
-            int idx = 0
-            while (true) {
-                byte read = is.read()
-                if (read == -1)
-                    throw new IOException("Stream closed")
-
-                if (!r && read == N)
-                    throw new IOException("Received N before R")
-                if (read == R) {
-                    if (r)
-                        throw new IOException("double R")
-                    r = true
-                    continue
-                }
-
-                if (r && !n) {
-                    if (read != N)
-                        throw new IOException("R not followed by N")
-                    n = true
-                    break
-                }
-                if (idx == 0x1 << 14)
-                    throw new IOException("Header too long")
-                tmp[idx++] = read
-            }
-
-            if (idx == 0)
-                break
-
-            String header = new String(tmp, 0, idx, StandardCharsets.US_ASCII)
-            log.fine("Read header $header")
-
-            int keyIdx = header.indexOf(":")
-            if (keyIdx < 1)
-                throw new IOException("Header key not found")
-            if (keyIdx == header.length())
-                throw new IOException("Header value not found")
-            String key = header.substring(0, keyIdx)
-            String value = header.substring(keyIdx + 1)
-            headers.put(key, value)
-        }
-        headers
-    }
-
 }

core/src/main/groovy/com/muwire/core/upload/UploadManager.groovy

@@ -12,6 +12,7 @@ import com.muwire.core.download.DownloadManager
 import com.muwire.core.download.Downloader
 import com.muwire.core.download.SourceDiscoveredEvent
 import com.muwire.core.files.FileManager
+import com.muwire.core.files.PersisterFolderService
 import com.muwire.core.mesh.Mesh
 import com.muwire.core.mesh.MeshManager
 
@@ -22,6 +23,7 @@ import net.i2p.data.Base64
 public class UploadManager {
     private final EventBus eventBus
     private final FileManager fileManager
+    private final PersisterFolderService persisterService
     private final MeshManager meshManager
     private final DownloadManager downloadManager
     private final MuWireSettings props
@@ -34,9 +36,11 @@ public class UploadManager {
 
     public UploadManager(EventBus eventBus, FileManager fileManager,
         MeshManager meshManager, DownloadManager downloadManager,
+        PersisterFolderService persisterService,
         MuWireSettings props) {
         this.eventBus = eventBus
         this.fileManager = fileManager
+        this.persisterService = persisterService
         this.meshManager = meshManager
         this.downloadManager = downloadManager
         this.props = props
@@ -162,7 +166,7 @@ public class UploadManager {
 
         InfoHash fullInfoHash
         if (downloader == null) {
-            fullInfoHash = sharedFiles.iterator().next().infoHash
+            fullInfoHash = persisterService.loadInfoHash(sharedFiles.iterator().next())
         } else {
             byte [] hashList = downloader.getInfoHash().getHashList()
             if (hashList != null && hashList.length > 0)

core/src/main/groovy/com/muwire/core/upload/Uploader.groovy

@@ -49,7 +49,7 @@ abstract class Uploader {
         final long now = System.currentTimeMillis()
         long interval = Math.max(1000, now - lastSpeedRead)
         lastSpeedRead = now;
-        int currSpeed = (int) (dataSinceLastRead * 1000.0 / interval)
+        int currSpeed = (int) (dataSinceLastRead * 1000.0d / interval)
         dataSinceLastRead = 0
 
         // normalize to speedArr.size

core/src/main/java/com/muwire/core/Constants.java

@@ -4,6 +4,8 @@ import net.i2p.crypto.SigType;
 
 public class Constants {
     public static final byte PERSONA_VERSION = (byte)1;
+    public static final String INVALID_NICKNAME_CHARS = "'\"();<>=@$%";
+    public static final int MAX_NICKNAME_LENGTH = 30;
     public static final byte FILE_CERT_VERSION = (byte)2;
     public static final int CHAT_VERSION = 1;
     
@@ -17,5 +19,8 @@ public class Constants {
     
     public static final int MAX_COMMENT_LENGTH = 0x1 << 15;
     
-    public static final long MAX_QUERY_AGE = 5 * 60 * 1000L; 
+    public static final long MAX_QUERY_AGE = 5 * 60 * 1000L;
+    
+    public static final int UPDATE_PORT = 2;
+    public static final int TRACKER_PORT = 3;
 }

core/src/main/java/com/muwire/core/DownloadedFile.java

@@ -10,9 +10,9 @@ public class DownloadedFile extends SharedFile {
 
     private final Set<Destination> sources;
 
-    public DownloadedFile(File file, InfoHash infoHash, int pieceSize, Set<Destination> sources)
+    public DownloadedFile(File file, byte[] root, int pieceSize, Set<Destination> sources)
     throws IOException {
-        super(file, infoHash, pieceSize);
+        super(file, root, pieceSize);
         this.sources = sources;
     }
 

core/src/main/java/com/muwire/core/InvalidNicknameException.java

@@ -0,0 +1,25 @@
+package com.muwire.core;
+
+public class InvalidNicknameException extends Exception {
+
+    public InvalidNicknameException() {
+    }
+
+    public InvalidNicknameException(String message) {
+        super(message);
+    }
+
+    public InvalidNicknameException(Throwable cause) {
+        super(cause);
+    }
+
+    public InvalidNicknameException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidNicknameException(String message, Throwable cause, boolean enableSuppression,
+            boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+}

core/src/main/java/com/muwire/core/Persona.java

@@ -7,6 +7,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import com.muwire.core.util.DataUtil;
+
 import net.i2p.crypto.DSAEngine;
 import net.i2p.data.Base64;
 import net.i2p.data.DataFormatException;
@@ -25,12 +27,15 @@ public class Persona {
     private volatile String base64;
     private volatile byte[] payload;
 
-    public Persona(InputStream personaStream) throws IOException, DataFormatException, InvalidSignatureException {
+    public Persona(InputStream personaStream) throws IOException, DataFormatException, InvalidSignatureException, InvalidNicknameException {
         version = (byte) (personaStream.read() & 0xFF);
         if (version != Constants.PERSONA_VERSION)
             throw new IOException("Unknown version "+version);
 
         name = new Name(personaStream);
+        if (!DataUtil.isValidName(name.name))
+            throw new InvalidNicknameException(name.name + " is not a valid nickname");
+        
         destination = Destination.create(personaStream);
         sig = new byte[SIG_LEN];
         DataInputStream dis = new DataInputStream(personaStream);
@@ -38,7 +43,7 @@ public class Persona {
         if (!verify(version, name, destination, sig))
             throw new InvalidSignatureException(getHumanReadableName() + " didn't verify");
     }
-
+    
     private static boolean verify(byte version, Name name, Destination destination, byte [] sig) 
     throws IOException, DataFormatException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -47,7 +52,7 @@ public class Persona {
         destination.writeBytes(baos);
         byte[] payload = baos.toByteArray();
         SigningPublicKey spk = destination.getSigningPublicKey();
-        Signature signature = new Signature(Constants.SIG_TYPE, sig);
+        Signature signature = new Signature(spk.getType(), sig);
         return DSAEngine.getInstance().verifySignature(signature, payload, spk);
     }
 

core/src/main/java/com/muwire/core/SharedFile.java

@@ -2,7 +2,10 @@ package com.muwire.core;
 
 import java.io.File;
 import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -16,44 +19,49 @@ import net.i2p.data.Base64;
 public class SharedFile {
 
     private final File file;
-    private final InfoHash infoHash;
+    private final byte[] root;
     private final int pieceSize;
 
     private final String cachedPath;
     private final long cachedLength;
-    
+
+    private String b64PathHash;
     private final String b64EncodedFileName;
-    private final String b64EncodedHashRoot;
-    private final List<String> b64EncodedHashList;
     
     private volatile String comment;
     private final Set<String> downloaders = Collections.synchronizedSet(new HashSet<>());
     private final Set<SearchEntry> searches = Collections.synchronizedSet(new HashSet<>());
+    private volatile boolean published;
+    private volatile long publishedTimestamp;
 
-    public SharedFile(File file, InfoHash infoHash, int pieceSize) throws IOException {
+    public SharedFile(File file, byte[] root, int pieceSize) throws IOException {
         this.file = file;
-        this.infoHash = infoHash;
+        this.root = root;
         this.pieceSize = pieceSize;
         this.cachedPath = file.getAbsolutePath();
         this.cachedLength = file.length();
         this.b64EncodedFileName = Base64.encode(DataUtil.encodei18nString(file.toString()));
-        this.b64EncodedHashRoot = Base64.encode(infoHash.getRoot());
-        
-        List<String> b64List = new ArrayList<String>();
-        byte[] tmp = new byte[32];
-        for (int i = 0; i < infoHash.getHashList().length / 32; i++) {
-            System.arraycopy(infoHash.getHashList(), i * 32, tmp, 0, 32);
-            b64List.add(Base64.encode(tmp));
-        }
-        this.b64EncodedHashList = b64List;
     }
 
     public File getFile() {
         return file;
     }
 
-    public InfoHash getInfoHash() {
-        return infoHash;
+    public byte[] getPathHash() throws NoSuchAlgorithmException {
+        MessageDigest digester = MessageDigest.getInstance("SHA-256");
+        digester.update(file.getAbsolutePath().getBytes());
+        return digester.digest();
+    }
+
+    public String getB64PathHash() throws NoSuchAlgorithmException {
+        if(b64PathHash == null){
+            b64PathHash = Base64.encode(getPathHash());
+        }
+        return b64PathHash;
+    }
+
+    public byte[] getRoot() {
+        return root;
     }
 
     public int getPieceSize() {
@@ -73,14 +81,6 @@ public class SharedFile {
         return b64EncodedFileName;
     }
     
-    public String getB64EncodedHashRoot() {
-        return b64EncodedHashRoot;
-    }
-    
-    public List<String> getB64EncodedHashList() {
-        return b64EncodedHashList;
-    }
-
     public String getCachedPath() {
         return cachedPath;
     }
@@ -116,10 +116,28 @@ public class SharedFile {
     public void addDownloader(String name) {
         downloaders.add(name);
     }
+    
+    public void publish(long timestamp) {
+        published = true;
+        publishedTimestamp = timestamp;
+    }
+    
+    public void unpublish() {
+        published = false;
+        publishedTimestamp = 0;
+    }
+    
+    public boolean isPublished() {
+        return published;
+    }
+    
+    public long getPublishedTimestamp() {
+        return publishedTimestamp;
+    }
 
     @Override
     public int hashCode() {
-        return file.hashCode() ^ infoHash.hashCode();
+        return file.hashCode() ^ Arrays.hashCode(root);
     }
 
     @Override
@@ -127,7 +145,7 @@ public class SharedFile {
         if (!(o instanceof SharedFile))
             return false;
         SharedFile other = (SharedFile)o;
-        return file.equals(other.file) && infoHash.equals(other.infoHash);
+        return file.equals(other.file) && Arrays.equals(root, other.root);
     }
     
     public static class SearchEntry {
@@ -141,6 +159,18 @@ public class SharedFile {
             this.query = query;
         }
         
+        public Persona getSearcher() {
+            return searcher;
+        }
+        
+        public long getTimestamp() {
+            return timestamp;
+        }
+        
+        public String getQuery() {
+            return query;
+        }
+        
         public int hashCode() {
             return Objects.hash(searcher) ^ Objects.hash(timestamp) ^ query.hashCode();
         }

core/src/main/java/com/muwire/core/filefeeds/Feed.java

@@ -0,0 +1,81 @@
+package com.muwire.core.filefeeds;
+
+import com.muwire.core.Persona;
+
+public class Feed {
+    
+    private final Persona publisher;
+    
+    private int updateInterval;
+    private long lastUpdated;
+    private volatile long lastUpdateAttempt;
+    private int itemsToKeep;
+    private boolean autoDownload;
+    private boolean sequential;
+    private FeedFetchStatus status;
+    
+    public Feed(Persona publisher) {
+        this.publisher = publisher;
+        this.status = FeedFetchStatus.IDLE;
+    }
+
+    public int getUpdateInterval() {
+        return updateInterval;
+    }
+
+    public void setUpdateInterval(int updateInterval) {
+        this.updateInterval = updateInterval;
+    }
+
+    public long getLastUpdated() {
+        return lastUpdated;
+    }
+
+    public void setLastUpdated(long lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    public int getItemsToKeep() {
+        return itemsToKeep;
+    }
+
+    public void setItemsToKeep(int itemsToKeep) {
+        this.itemsToKeep = itemsToKeep;
+    }
+
+    public boolean isAutoDownload() {
+        return autoDownload;
+    }
+
+    public void setAutoDownload(boolean autoDownload) {
+        this.autoDownload = autoDownload;
+    }
+
+    public Persona getPublisher() {
+        return publisher;
+    }
+
+    public void setStatus(FeedFetchStatus status) {
+        this.status = status;
+    }
+    
+    public FeedFetchStatus getStatus() {
+        return status;
+    }
+    
+    public void setSequential(boolean sequential) {
+        this.sequential = sequential;
+    }
+    
+    public boolean isSequential() {
+        return sequential;
+    }
+    
+    public void setLastUpdateAttempt(long lastUpdateAttempt) {
+        this.lastUpdateAttempt = lastUpdateAttempt;
+    }
+    
+    public long getLastUpdateAttempt() {
+        return lastUpdateAttempt;
+    }
+}

core/src/main/java/com/muwire/core/filefeeds/FeedFetchStatus.java

@@ -0,0 +1,19 @@
+package com.muwire.core.filefeeds;
+
+public enum FeedFetchStatus {
+    IDLE(false), 
+    CONNECTING(true), 
+    FETCHING(true), 
+    FINISHED(false), 
+    FAILED(false);
+    
+    private final boolean active;
+    
+    FeedFetchStatus(boolean active) {
+        this.active = active;
+    }
+    
+    public boolean isActive() {
+        return active;
+    }
+}

core/src/main/java/com/muwire/core/filefeeds/FeedItem.java

@@ -0,0 +1,79 @@
+package com.muwire.core.filefeeds;
+
+import java.util.Objects;
+
+import com.muwire.core.InfoHash;
+import com.muwire.core.Persona;
+
+public class FeedItem {
+
+    private final Persona publisher;
+    private final long timestamp;
+    private final String name;
+    private final long size;
+    private final int pieceSize;
+    private final InfoHash infoHash;
+    private final int certificates;
+    private final String comment;
+    
+    public FeedItem(Persona publisher, long timestamp, String name, long size, int pieceSize, InfoHash infoHash,
+            int certificates, String comment) {
+        super();
+        this.publisher = publisher;
+        this.timestamp = timestamp;
+        this.name = name;
+        this.size = size;
+        this.pieceSize = pieceSize;
+        this.infoHash = infoHash;
+        this.certificates = certificates;
+        this.comment = comment;
+    }
+
+    public Persona getPublisher() {
+        return publisher;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    public int getPieceSize() {
+        return pieceSize;
+    }
+
+    public InfoHash getInfoHash() {
+        return infoHash;
+    }
+
+    public int getCertificates() {
+        return certificates;
+    }
+
+    public String getComment() {
+        return comment;
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(publisher, timestamp, name, infoHash);
+    }
+    
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof FeedItem))
+            return false;
+        FeedItem other = (FeedItem)o;
+        return Objects.equals(publisher, other.publisher) &&
+                timestamp == other.timestamp &&
+                Objects.equals(name, other.name) &&
+                Objects.equals(infoHash, other.infoHash);
+    }
+}

core/src/main/java/com/muwire/core/filefeeds/InvalidFeedItemException.java

@@ -0,0 +1,30 @@
+package com.muwire.core.filefeeds;
+
+public class InvalidFeedItemException extends Exception {
+
+    public InvalidFeedItemException() {
+        super();
+    }
+
+    public InvalidFeedItemException(String message, Throwable cause, boolean enableSuppression,
+            boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+        // TODO Auto-generated constructor stub
+    }
+
+    public InvalidFeedItemException(String message, Throwable cause) {
+        super(message, cause);
+        // TODO Auto-generated constructor stub
+    }
+
+    public InvalidFeedItemException(String message) {
+        super(message);
+        // TODO Auto-generated constructor stub
+    }
+
+    public InvalidFeedItemException(Throwable cause) {
+        super(cause);
+        // TODO Auto-generated constructor stub
+    }
+
+}

core/src/main/java/com/muwire/core/util/DataUtil.java

@@ -58,9 +58,9 @@ public class DataUtil {
         if (header.length != 3)
             throw new IllegalArgumentException("header length $header.length");
 
-        return (((int)(header[0] & 0x7F)) << 16) |
-                (((int)(header[1] & 0xFF) << 8)) |
-                ((int)header[2] & 0xFF);
+        return ((header[0] & 0x7F) << 16) |
+                ((header[1] & 0xFF) << 8) |
+                (header[2] & 0xFF);
     }
 
     public static String readi18nString(byte [] encoded) {
@@ -174,7 +174,7 @@ public class DataUtil {
                 clean.setAccessible(true);
                 clean.invoke(cleaner.invoke(cb));
             } else {
-                Class unsafeClass;
+                Class<?> unsafeClass;
                 try {
                     unsafeClass = Class.forName("sun.misc.Unsafe");
                 } catch(Exception ex) {
@@ -216,4 +216,13 @@ public class DataUtil {
         Signature sig = DSAEngine.getInstance().sign(payload, spk);
         return sig.getData();
     }
+    
+    public static boolean isValidName(String name) {
+        if (name.length() > Constants.MAX_NICKNAME_LENGTH)
+            return false;
+        for (int i = 0; i < Constants.INVALID_NICKNAME_CHARS.length(); i++)
+            if (name.indexOf(Constants.INVALID_NICKNAME_CHARS.charAt(i)) >= 0)
+                return false;
+        return true;
+    }
 }

core/src/main/resources/defaults/i2p.properties

@@ -0,0 +1,8 @@
+inbound.nickname=MuWire
+outbound.nickname=MuWire
+inbound.length=3
+inbound.quantity=4
+outbound.length=3
+outbound.quantity=4
+i2cp.tcp.host=127.0.0.1
+i2cp.tcp.port=7654

core/src/test/groovy/com/muwire/core/files/FileManagerTest.groovy

@@ -39,13 +39,13 @@ class FileManagerTest {
     @Test
     void testHash1Result() {
         File f = new File("a b.c")
-        InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih, 0)
+        byte [] root = new byte[32]
+        SharedFile sf = new SharedFile(f,root, 0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
         UUID uuid = UUID.randomUUID()
-        SearchEvent se = new SearchEvent(searchHash: ih.getRoot(), uuid: uuid)
+        SearchEvent se = new SearchEvent(searchHash: root, uuid: uuid)
 
         manager.onSearchEvent(se)
         Thread.sleep(20)
@@ -58,14 +58,14 @@ class FileManagerTest {
 
     @Test
     void testHash2Results() {
-        InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(new File("a b.c"), ih, 0)
-        SharedFile sf2 = new SharedFile(new File("d e.f"), ih, 0)
+        byte [] root = new byte[32]
+        SharedFile sf1 = new SharedFile(new File("a b.c"), root, 0)
+        SharedFile sf2 = new SharedFile(new File("d e.f"), root, 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf1)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf2)
 
         UUID uuid = UUID.randomUUID()
-        SearchEvent se = new SearchEvent(searchHash: ih.getRoot(), uuid: uuid)
+        SearchEvent se = new SearchEvent(searchHash: root, uuid: uuid)
 
         manager.onSearchEvent(se)
         Thread.sleep(20)
@@ -81,7 +81,7 @@ class FileManagerTest {
     void testHash0Results() {
         File f = new File("a b.c")
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih, 0)
+        SharedFile sf = new SharedFile(f,ih.getRoot(), 0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
@@ -95,7 +95,7 @@ class FileManagerTest {
     void testKeyword1Result() {
         File f = new File("a b.c")
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih,0)
+        SharedFile sf = new SharedFile(f,ih.getRoot(),0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
@@ -113,12 +113,12 @@ class FileManagerTest {
     void testKeyword2Results() {
         File f1 = new File("a b.c")
         InfoHash ih1 = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(f1, ih1, 0)
+        SharedFile sf1 = new SharedFile(f1, ih1.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf1)
 
         File f2 = new File("c d.e")
         InfoHash ih2 = InfoHash.fromHashList(new byte[64])
-        SharedFile sf2 = new SharedFile(f2, ih2, 0)
+        SharedFile sf2 = new SharedFile(f2, ih2.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf2)
 
         UUID uuid = UUID.randomUUID()
@@ -136,7 +136,7 @@ class FileManagerTest {
     void testKeyword0Results() {
         File f = new File("a b.c")
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf = new SharedFile(f,ih,0)
+        SharedFile sf = new SharedFile(f,ih.getRoot(),0)
         FileHashedEvent fhe = new FileHashedEvent(sharedFile: sf)
         manager.onFileHashedEvent(fhe)
 
@@ -149,8 +149,8 @@ class FileManagerTest {
     @Test
     void testRemoveFileExistingHash() {
         InfoHash ih = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(new File("a b.c"), ih, 0)
-        SharedFile sf2 = new SharedFile(new File("d e.f"), ih, 0)
+        SharedFile sf1 = new SharedFile(new File("a b.c"), ih.getRoot(), 0)
+        SharedFile sf2 = new SharedFile(new File("d e.f"), ih.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf1)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile : sf2)
 
@@ -167,12 +167,12 @@ class FileManagerTest {
     void testRemoveFile() {
         File f1 = new File("a b.c")
         InfoHash ih1 = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(f1, ih1, 0)
+        SharedFile sf1 = new SharedFile(f1, ih1.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf1)
 
         File f2 = new File("c d.e")
         InfoHash ih2 = InfoHash.fromHashList(new byte[64])
-        SharedFile sf2 = new SharedFile(f2, ih2, 0)
+        SharedFile sf2 = new SharedFile(f2, ih2.getRoot(), 0)
         manager.onFileLoadedEvent new FileLoadedEvent(loadedFile: sf2)
 
         manager.onFileUnsharedEvent new FileUnsharedEvent(deleted : true, unsharedFile: sf2)
@@ -198,7 +198,7 @@ class FileManagerTest {
         comment = Base64.encode(DataUtil.encodei18nString(comment))
         File f1 = new File("MuWire-0.5.10.AppImage")
         InfoHash ih1 = InfoHash.fromHashList(new byte[32])
-        SharedFile sf1 = new SharedFile(f1, ih1, 0)
+        SharedFile sf1 = new SharedFile(f1, ih1.getRoot(), 0)
         sf1.setComment(comment)
         
         manager.onFileLoadedEvent(new FileLoadedEvent(loadedFile : sf1))
@@ -206,7 +206,7 @@ class FileManagerTest {
         
         File f2 = new File("MuWire-0.6.0.AppImage")
         InfoHash ih2 = InfoHash.fromHashList(new byte[64])
-        SharedFile sf2 = new SharedFile(f2, ih2, 0)
+        SharedFile sf2 = new SharedFile(f2, ih2.getRoot(), 0)
         sf2.setComment(comment)
         
         manager.onFileLoadedEvent(new FileLoadedEvent(loadedFile : sf2))

core/src/test/groovy/com/muwire/core/files/HasherServiceTest.groovy

@@ -45,7 +45,7 @@ class HasherServiceTest {
         def hashed = listener.poll()
         assert hashed instanceof FileHashedEvent
         assert hashed.sharedFile.file == f.getCanonicalFile()
-        assert hashed.sharedFile.infoHash != null
+        assert hashed.sharedFile.root != null
         assert listener.isEmpty()
     }
 

core/src/test/groovy/com/muwire/core/files/PersisterServiceLoadingTest.groovy

@@ -85,7 +85,7 @@ class PersisterServiceLoadingTest {
         def loadedFile = listener.publishedFiles[0]
         assert loadedFile != null
         assert loadedFile.file == sharedFile1.getCanonicalFile()
-        assert loadedFile.infoHash == ih1
+        assert loadedFile.root == ih1.getRoot()
     }
 
     private static String getSharedFileJsonName(File sharedFile) {
@@ -128,7 +128,7 @@ class PersisterServiceLoadingTest {
         def loadedFile = listener.publishedFiles[0]
         assert loadedFile != null
         assert loadedFile.file == sharedFile1.getCanonicalFile()
-        assert loadedFile.infoHash == ih1
+        assert loadedFile.root == ih1.getRoot()
     }
 
     @Test
@@ -169,10 +169,10 @@ class PersisterServiceLoadingTest {
         assert listener.publishedFiles.size() == 2
         def loadedFile1 = listener.publishedFiles[0]
         assert loadedFile1.file == sharedFile1.getCanonicalFile()
-        assert loadedFile1.infoHash == ih1
+        assert loadedFile1.root == ih1.getRoot()
         def loadedFile2 = listener.publishedFiles[1]
         assert loadedFile2.file == sharedFile2.getCanonicalFile()
-        assert loadedFile2.infoHash == ih2
+        assert loadedFile2.root == ih2.getRoot()
     }
 
     @Test

core/src/test/groovy/com/muwire/core/files/PersisterServiceSavingTest.groovy

@@ -2,6 +2,7 @@ package com.muwire.core.files
 
 import org.junit.After
 import org.junit.Before
+import org.junit.Ignore
 import org.junit.Test
 
 import com.muwire.core.Destinations
@@ -16,6 +17,7 @@ import groovy.json.JsonSlurper
 import net.i2p.data.Base32
 import net.i2p.data.Base64
 
+@Ignore
 class PersisterServiceSavingTest {
 
     File f

docker/rootfs/etc/cont-init.d/00-app-user-map.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/with-contenv sh
+
+#
+# Add the app user to the password and group databases.  This is needed just to
+# make sure that mapping between the user/group ID and its name is possible.
+#
+
+set -e # Exit immediately if a command exits with a non-zero status.
+set -u # Treat unset variables as an error.
+
+cp /defaults/passwd /etc/passwd
+cp /defaults/group /etc/group
+cp /defaults/shadow /etc/shadow
+chown root:shadow /etc/shadow
+chmod 640 /etc/shadow
+
+echo "$APP_USER:x:$USER_ID:$GROUP_ID::${APP_HOME:-/dev/null}:/sbin/nologin" >> /etc/passwd
+echo "$APP_USER:x:$GROUP_ID:" >> /etc/group
+
+# Make sure APP_HOME is editable by the user
+if [[ -n "$APP_HOME" ]] ; then
+    chown -R "$APP_USER" "$APP_HOME"
+    chmod -R u+rw "$APP_HOME"
+fi
+
+# vim:ft=sh:ts=4:sw=4:et:sts=4

docker/rootfs/muwire/.MuWire/MuWire.properties

@@ -0,0 +1,34 @@
+#This file is UTF-8
+#Tue Jan 14 12:08:47 GMT 2020
+meshExpiration=60
+autoDownloadUpdate=true
+hostHopelessInterval=1440
+uploadSlotsPerUser=-1
+downloadLocation=/output
+allowTrustLists=true
+embeddedRouter=false
+incompleteLocation=/incompletes
+outBw=128
+searchExtraHop=false
+shareHiddenFiles=false
+advertiseChat=true
+totalUploadSlots=-1
+hostClearInterval=15
+searchComments=true
+downloadSequentialRatio=0.8
+maxChatConnectios=-1
+trustListInterval=1
+crawlerResponse=REGISTERED
+browseFiles=true
+lastUpdateCheck=1579003533112
+hostRejectInterval=1
+inBw=256
+leaf=false
+updateCheckInterval=24
+plugin=false
+downloadRetryInterval=60
+speedSmoothSeconds=60
+allowUntrusted=true
+shareDownloadedFiles=true
+startChatServer=false
+updateType=jar

docker/rootfs/muwire/.MuWire/i2p.properties

@@ -0,0 +1 @@
+i2cp.tcp.host=172.17.0.1

docker/rootfs/startapp.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Explicitly define HOME otherwise it might not have been set
+export HOME=/muwire
+
+echo "Starting MuWire"
+exec /muwire/bin/MuWire

gradle.properties

@@ -1,6 +1,6 @@
 group         = com.muwire
-version       = 0.6.8
-i2pVersion    = 0.9.44
+version       = 0.7.0
+i2pVersion    = 0.9.46
 groovyVersion = 2.4.15
 slf4jVersion  = 1.7.25
 spockVersion  = 1.1-groovy-2.4
@@ -8,14 +8,16 @@ grailsVersion=4.0.0
 gorm.version=7.0.2.RELEASE
 griffonEnv=prod
 
+# javac properties
 sourceCompatibility=1.8
 targetCompatibility=1.8
+compilerArgs=-Xlint:unchecked,cast,path,divzero,empty,path,finally,overrides
 
 # plugin properties
 author = zab@mail.i2p
 signer = zab@mail.i2p
 keystorePassword=changeit
 websiteURL=http://muwire.i2p
-updateURLsu3=http://muwire.i2p/MuWire.su3
+updateURLsu3=http://muwire.i2p/MuWire-update.su3
 
 pack200=true

gui/griffon-app/conf/Config.groovy

@@ -126,4 +126,24 @@ mvcGroups {
         view = 'com.muwire.gui.ChatMonitorView'
         controller = 'com.muwire.gui.ChatMonitorController'
     }
+    'feed-configuration' {
+        model = 'com.muwire.gui.FeedConfigurationModel'
+        view = 'com.muwire.gui.FeedConfigurationView'
+        controller = 'com.muwire.gui.FeedConfigurationController'
+    }
+    'watched-directory' {
+        model = 'com.muwire.gui.WatchedDirectoryModel'
+        view = 'com.muwire.gui.WatchedDirectoryView'
+        controller = 'com.muwire.gui.WatchedDirectoryController'
+    }
+    'sign' {
+        model = 'com.muwire.gui.SignModel'
+        view = 'com.muwire.gui.SignView'
+        controller = 'com.muwire.gui.SignController'
+    }
+    'wizard' {
+        model = 'com.muwire.gui.wizard.WizardModel'
+        view = 'com.muwire.gui.wizard.WizardView'
+        controller = 'com.muwire.gui.wizard.WizardController'
+    }
 }

gui/griffon-app/controllers/com/muwire/gui/AdvancedSharingController.groovy

@@ -7,6 +7,7 @@ import griffon.metadata.ArtifactProviderFor
 import javax.annotation.Nonnull
 
 import com.muwire.core.Core
+import com.muwire.core.files.directories.UISyncDirectoryEvent
 
 @ArtifactProviderFor(GriffonController)
 class AdvancedSharingController {
@@ -14,4 +15,25 @@ class AdvancedSharingController {
     AdvancedSharingModel model
     @MVCMember @Nonnull
     AdvancedSharingView view
+    
+    @ControllerAction
+    void configure() {
+        def wd = view.selectedWatchedDirectory()
+        if (wd == null)
+            return
+            
+        def params = [:]
+        params['core'] = model.core
+        params['directory'] = wd
+        mvcGroup.createMVCGroup("watched-directory",params)
+    }
+    
+    @ControllerAction
+    void sync() {
+        def wd = view.selectedWatchedDirectory()
+        if (wd == null)
+            return
+        def event = new UISyncDirectoryEvent(directory : wd.directory)
+        model.core.eventBus.publish(event)
+    }
 }

gui/griffon-app/controllers/com/muwire/gui/BrowseController.groovy

@@ -113,7 +113,9 @@ class BrowseController {
             return
         
         def params = [:]
-        params['result'] = result
+        params['host'] = result.getSender()
+        params['infoHash'] = result.getInfohash()
+        params['name'] = result.getName()
         params['core'] = core
         mvcGroup.createMVCGroup("fetch-certificates", params)
     }

gui/griffon-app/controllers/com/muwire/gui/FeedConfigurationController.groovy

@@ -0,0 +1,36 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import javax.annotation.Nonnull
+
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
+
+@ArtifactProviderFor(GriffonController)
+class FeedConfigurationController {
+    @MVCMember @Nonnull
+    FeedConfigurationModel model
+    @MVCMember @Nonnull
+    FeedConfigurationView view
+
+    @ControllerAction
+    void save() {
+        
+        model.feed.setAutoDownload(view.autoDownloadCheckbox.model.isSelected())
+        model.feed.setSequential(view.sequentialCheckbox.model.isSelected())
+        model.feed.setItemsToKeep(Integer.parseInt(view.itemsToKeepField.text))
+        model.feed.setUpdateInterval(Integer.parseInt(view.updateIntervalField.text) * 60000)
+        
+        model.core.eventBus.publish(new UIFeedConfigurationEvent(feed : model.feed))
+        
+        cancel()    
+    }
+    
+    @ControllerAction
+    void cancel() {
+        view.dialog.setVisible(false)
+        mvcGroup.destroy()
+    }
+}

gui/griffon-app/controllers/com/muwire/gui/FetchCertificatesController.groovy

@@ -28,7 +28,7 @@ class FetchCertificatesController {
         core.eventBus.with { 
             register(CertificateFetchEvent.class, this)
             register(CertificateFetchedEvent.class, this)
-            publish(new UIFetchCertificatesEvent(host : model.result.sender, infoHash : model.result.infohash))
+            publish(new UIFetchCertificatesEvent(host : model.host, infoHash : model.infoHash))
         }
     }
     

gui/griffon-app/controllers/com/muwire/gui/MainFrameController.groovy

@@ -3,15 +3,11 @@ package com.muwire.gui
 import griffon.core.GriffonApplication
 import griffon.core.artifact.GriffonController
 import griffon.core.controller.ControllerAction
-import griffon.core.mvc.MVCGroup
-import griffon.core.mvc.MVCGroupConfiguration
 import griffon.inject.MVCMember
 import griffon.metadata.ArtifactProviderFor
-import groovy.json.StringEscapeUtils
 import net.i2p.crypto.DSAEngine
 import net.i2p.data.Base64
 import net.i2p.data.Signature
-import net.i2p.data.SigningPrivateKey
 
 import java.awt.Desktop
 import java.awt.Toolkit
@@ -30,15 +26,18 @@ import com.muwire.core.Persona
 import com.muwire.core.SharedFile
 import com.muwire.core.SplitPattern
 import com.muwire.core.download.Downloader
-import com.muwire.core.download.DownloadStartedEvent
 import com.muwire.core.download.UIDownloadCancelledEvent
-import com.muwire.core.download.UIDownloadEvent
 import com.muwire.core.download.UIDownloadPausedEvent
 import com.muwire.core.download.UIDownloadResumedEvent
 import com.muwire.core.filecert.UICreateCertificateEvent
-import com.muwire.core.files.DirectoryUnsharedEvent
+import com.muwire.core.filefeeds.Feed
+import com.muwire.core.filefeeds.FeedItem
+import com.muwire.core.filefeeds.UIDownloadFeedItemEvent
+import com.muwire.core.filefeeds.UIFeedDeletedEvent
+import com.muwire.core.filefeeds.UIFeedUpdateEvent
+import com.muwire.core.filefeeds.UIFilePublishedEvent
+import com.muwire.core.filefeeds.UIFileUnpublishedEvent
 import com.muwire.core.files.FileUnsharedEvent
-import com.muwire.core.files.UIPersistFilesEvent
 import com.muwire.core.search.QueryEvent
 import com.muwire.core.search.SearchEvent
 import com.muwire.core.trust.RemoteTrustList
@@ -371,7 +370,6 @@ class MainFrameController {
         sf.each {  
             core.eventBus.publish(new FileUnsharedEvent(unsharedFile : it))
         }
-        core.eventBus.publish(new UIPersistFilesEvent())
     }
     
     @ControllerAction
@@ -514,6 +512,105 @@ class MainFrameController {
         clipboard.setContents(selection, null)
     }
     
+    @ControllerAction
+    void publish() {
+        def selectedFiles = view.selectedSharedFiles()
+        if (selectedFiles == null || selectedFiles.isEmpty())
+            return
+        
+        if (model.publishButtonText == "Unpublish") {
+            selectedFiles.each { 
+                it.unpublish()
+                model.core.eventBus.publish(new UIFileUnpublishedEvent(sf : it))
+            }
+        } else {
+            long now = System.currentTimeMillis()
+            selectedFiles.stream().filter({!it.isPublished()}).forEach({
+                it.publish(now)
+                model.core.eventBus.publish(new UIFilePublishedEvent(sf : it))
+            })
+        }
+        view.refreshSharedFiles()
+    }
+    
+    @ControllerAction
+    void updateFileFeed() {
+        Feed feed = view.selectedFeed()
+        if (feed == null)
+            return
+        model.core.eventBus.publish(new UIFeedUpdateEvent(host: feed.getPublisher()))
+    }
+    
+    @ControllerAction
+    void unsubscribeFileFeed() {
+        Feed feed = view.selectedFeed()
+        if (feed == null)
+            return
+        model.core.eventBus.publish(new UIFeedDeletedEvent(host : feed.getPublisher()))
+        runInsideUIAsync {
+            model.feeds.remove(feed)
+            model.feedItems.clear()
+            view.refreshFeeds()
+        }
+    }
+    
+    @ControllerAction
+    void configureFileFeed() {
+        Feed feed = view.selectedFeed()
+        if (feed == null)
+            return
+        
+        def params = [:]
+        params['core'] = core
+        params['feed'] = feed
+        mvcGroup.createMVCGroup("feed-configuration", params)
+    }
+    
+    @ControllerAction
+    void downloadFeedItem() {
+        List<FeedItem> items = view.selectedFeedItems()
+        if (items == null || items.isEmpty())
+            return
+        Feed f = model.core.getFeedManager().getFeed(items.get(0).getPublisher())
+        items.each { 
+            if (!model.canDownload(it.getInfoHash()))
+                return
+            File target = new File(application.context.get("muwire-settings").downloadLocation, it.getName())
+            model.core.eventBus.publish(new UIDownloadFeedItemEvent(item : it, target : target, sequential : f.isSequential()))
+        }
+        view.showDownloadsWindow.call()
+    }
+    
+    @ControllerAction
+    void viewFeedItemComment() {
+        List<FeedItem> items = view.selectedFeedItems()
+        if (items == null || items.size() != 1)
+            return
+        FeedItem item = items.get(0)
+        
+        String groupId = Base64.encode(item.getInfoHash().getRoot())
+        Map<String, Object> params = new HashMap<>()
+        params['text'] = DataUtil.readi18nString(Base64.decode(item.getComment()))
+        params['name'] = item.getName()
+        
+        mvcGroup.createMVCGroup("show-comment", groupId, params)
+    }
+    
+    @ControllerAction
+    void viewFeedItemCertificates() {
+        List<FeedItem> items = view.selectedFeedItems()
+        if (items == null || items.size() != 1)
+            return
+        FeedItem item = items.get(0)
+        
+        def params = [:]
+        params['core'] = core
+        params['host'] = item.getPublisher()
+        params['infoHash'] = item.getInfoHash()
+        params['name'] = item.getName()
+        mvcGroup.createMVCGroup("fetch-certificates", params)
+    }
+    
     void startChat(Persona p) {
         if (!mvcGroup.getChildrenGroups().containsKey(p.getHumanReadableName())) {
             def params = [:]
@@ -534,4 +631,4 @@ class MainFrameController {
             core = e.getNewValue()
         })
     }
-}
+}

gui/griffon-app/controllers/com/muwire/gui/OptionsController.groovy

@@ -32,22 +32,14 @@ class OptionsController {
 
         def i2pProps = core.i2pOptions
 
-        text = view.inboundLengthField.text
-        model.inboundLength = text
-        i2pProps["inbound.length"] = text
-
-        text = view.inboundQuantityField.text
-        model.inboundQuantity = text
-        i2pProps["inbound.quantity"] = text
-
-        text = view.outboundQuantityField.text
-        model.outboundQuantity = text
-        i2pProps["outbound.quantity"] = text
-
-        text = view.outboundLengthField.text
-        model.outboundLength = text
-        i2pProps["outbound.length"] = text
-
+        int tunnelLength = view.tunnelLengthSlider.value
+        i2pProps["inbound.length"] = String.valueOf(tunnelLength)
+        i2pProps["outbound.length"] = String.valueOf(tunnelLength)
+        
+        int tunnelQuantity = view.tunnelQuantitySlider.value
+        i2pProps["inbound.quantity"] = String.valueOf(tunnelQuantity)
+        i2pProps["outbound.quantity"] = String.valueOf(tunnelQuantity)
+        
         if (settings.embeddedRouter) {
             text = view.i2pNTCPPortField.text
             model.i2pNTCPPort = text
@@ -104,6 +96,10 @@ class OptionsController {
         model.browseFiles = browseFiles
         settings.browseFiles = browseFiles
         
+        boolean allowTracking = view.allowTrackingCheckbox.model.isSelected()
+        model.allowTracking = allowTracking
+        settings.allowTracking = allowTracking
+        
         text = view.speedSmoothSecondsField.text
         model.speedSmoothSeconds = Integer.valueOf(text)
         settings.speedSmoothSeconds = Integer.valueOf(text)
@@ -122,7 +118,38 @@ class OptionsController {
             model.outBw = text
             settings.outBw = Integer.valueOf(text)
         }
+        
+        // feed saving
+        
+        boolean fileFeed = view.fileFeedCheckbox.model.isSelected()
+        model.fileFeed = fileFeed
+        settings.fileFeed = fileFeed
+        
+        boolean advertiseFeed = view.advertiseFeedCheckbox.model.isSelected()
+        model.advertiseFeed = advertiseFeed
+        settings.advertiseFeed = advertiseFeed
+        
+        boolean autoPublishSharedFiles = view.autoPublishSharedFilesCheckbox.model.isSelected()
+        model.autoPublishSharedFiles = autoPublishSharedFiles
+        settings.autoPublishSharedFiles = autoPublishSharedFiles
+        
+        boolean defaultFeedAutoDownload = view.defaultFeedAutoDownloadCheckbox.model.isSelected()
+        model.defaultFeedAutoDownload = defaultFeedAutoDownload
+        settings.defaultFeedAutoDownload = defaultFeedAutoDownload
+        
+        boolean defaultFeedSequential = view.defaultFeedSequentialCheckbox.model.isSelected()
+        model.defaultFeedSequential = defaultFeedSequential
+        settings.defaultFeedSequential = defaultFeedSequential
+        
+        String defaultFeedItemsToKeep = view.defaultFeedItemsToKeepField.text
+        model.defaultFeedItemsToKeep = defaultFeedItemsToKeep
+        settings.defaultFeedItemsToKeep = Integer.parseInt(defaultFeedItemsToKeep)
+        
+        String defaultFeedUpdateInterval = view.defaultFeedUpdateIntervalField.text
+        model.defaultFeedUpdateInterval = defaultFeedUpdateInterval
+        settings.defaultFeedUpdateInterval = Integer.parseInt(defaultFeedUpdateInterval)
 
+        // trust saving
 
         boolean onlyTrusted = view.allowUntrustedCheckbox.model.isSelected()
         model.onlyTrusted = onlyTrusted

gui/griffon-app/controllers/com/muwire/gui/SearchTabController.groovy

@@ -12,6 +12,8 @@ import javax.swing.JOptionPane
 import com.muwire.core.Core
 import com.muwire.core.Persona
 import com.muwire.core.download.UIDownloadEvent
+import com.muwire.core.filefeeds.Feed
+import com.muwire.core.filefeeds.UIFeedConfigurationEvent
 import com.muwire.core.search.UIResultEvent
 import com.muwire.core.trust.TrustEvent
 import com.muwire.core.trust.TrustLevel
@@ -107,6 +109,22 @@ class SearchTabController {
         mvcGroup.createMVCGroup("browse", groupId, params)
     }
     
+    @ControllerAction
+    void subscribe() {
+        def sender = view.selectedSender()
+        if (sender == null)
+            return
+
+        Feed feed = new Feed(sender)
+        feed.setAutoDownload(core.muOptions.defaultFeedAutoDownload)
+        feed.setSequential(core.muOptions.defaultFeedSequential)
+        feed.setItemsToKeep(core.muOptions.defaultFeedItemsToKeep)
+        feed.setUpdateInterval(core.muOptions.defaultFeedUpdateInterval * 60 * 1000)
+        
+        core.eventBus.publish(new UIFeedConfigurationEvent(feed : feed, newFeed: true))  
+        mvcGroup.parentGroup.view.showFeedsWindow.call()          
+    }
+    
     @ControllerAction
     void chat() {
         def sender = view.selectedSender()
@@ -139,7 +157,9 @@ class SearchTabController {
             return
 
         def params = [:]
-        params['result'] = event
+        params['host'] = event.getSender()
+        params['infoHash'] = event.getInfohash()
+        params['name'] = event.getName()
         params['core'] = core
         mvcGroup.createMVCGroup("fetch-certificates", params)
     }

gui/griffon-app/controllers/com/muwire/gui/SignController.groovy

@@ -0,0 +1,50 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import net.i2p.crypto.DSAEngine
+import net.i2p.data.Base64
+
+import java.awt.Toolkit
+import java.awt.datatransfer.StringSelection
+import java.nio.charset.StandardCharsets
+
+import javax.annotation.Nonnull
+import javax.swing.JOptionPane
+
+import com.muwire.core.Constants
+import com.muwire.core.Core
+import com.muwire.core.util.DataUtil
+
+@ArtifactProviderFor(GriffonController)
+class SignController {
+    
+    Core core
+    
+    @MVCMember @Nonnull
+    SignView view
+
+    @ControllerAction
+    void sign() {
+        String plain = view.plainTextArea.getText()
+        byte[] payload = plain.trim().getBytes(StandardCharsets.UTF_8)
+        def sig = DSAEngine.getInstance().sign(payload, core.spk)
+        view.signedTextArea.setText(Base64.encode(sig.data))
+    }
+    
+    @ControllerAction
+    void copy() {
+        String signed = view.signedTextArea.getText()
+        StringSelection selection = new StringSelection(signed)
+        def clipboard = Toolkit.getDefaultToolkit().getSystemClipboard()
+        clipboard.setContents(selection, null)
+    }
+    
+    @ControllerAction
+    void close() {
+        view.dialog.setVisible(false)
+        mvcGroup.destroy()
+    }
+}

gui/griffon-app/controllers/com/muwire/gui/WatchedDirectoryController.groovy

@@ -0,0 +1,33 @@
+package com.muwire.gui
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import javax.annotation.Nonnull
+
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+
+@ArtifactProviderFor(GriffonController)
+class WatchedDirectoryController {
+    @MVCMember @Nonnull
+    WatchedDirectoryModel model
+    @MVCMember @Nonnull
+    WatchedDirectoryView view
+
+    @ControllerAction
+    void save() {
+        def event = new WatchedDirectoryConfigurationEvent(
+            directory : model.directory.directory,
+            autoWatch : view.autoWatchCheckbox.model.isSelected(),
+            syncInterval : Integer.parseInt(view.syncIntervalField.text))
+        model.core.eventBus.publish(event)
+        cancel()
+    }
+    
+    @ControllerAction
+    void cancel() {
+        view.dialog.setVisible(false)
+        mvcGroup.destroy()
+    }
+}

gui/griffon-app/controllers/com/muwire/gui/wizard/WizardController.groovy

@@ -0,0 +1,49 @@
+package com.muwire.gui.wizard
+
+import griffon.core.artifact.GriffonController
+import griffon.core.controller.ControllerAction
+import griffon.inject.MVCMember
+import griffon.metadata.ArtifactProviderFor
+import javax.annotation.Nonnull
+import javax.swing.JOptionPane
+
+@ArtifactProviderFor(GriffonController)
+class WizardController {
+    @MVCMember @Nonnull
+    WizardModel model
+    @MVCMember @Nonnull
+    WizardView view
+
+    @ControllerAction
+    void previous() {
+        model.currentStep--
+        view.updateLayout()
+    }
+    
+    @ControllerAction
+    void next() {
+        def errors = model.steps[model.currentStep].validate()
+        if (errors != null && !errors.isEmpty()) {
+            String errorMessage = String.join("\n", errors)
+            JOptionPane.showMessageDialog(model.parent, errorMessage, "Invalid Input", JOptionPane.ERROR_MESSAGE)
+        } else {
+            model.currentStep++
+            view.updateLayout()
+        }
+    }
+    
+    @ControllerAction
+    void finish() {
+        model.steps.each { 
+            it.apply(model.muSettings, model.i2pProps)
+        }
+        model.finished['applied'] = true
+        view.hide()
+    }
+    
+    @ControllerAction
+    void cancel() {
+        model.finished['applied'] = false
+        view.hide()
+    }
+}

gui/griffon-app/lifecycle/Ready.groovy

@@ -6,10 +6,12 @@ import net.i2p.util.SystemVersion
 
 import org.codehaus.griffon.runtime.core.AbstractLifecycleHandler
 
+import com.muwire.core.Constants
 import com.muwire.core.Core
 import com.muwire.core.MuWireSettings
 import com.muwire.core.UILoadedEvent
 import com.muwire.core.files.FileSharedEvent
+import com.muwire.core.util.DataUtil
 
 import javax.annotation.Nonnull
 import javax.inject.Inject
@@ -44,56 +46,41 @@ class Ready extends AbstractLifecycleHandler {
             propsFile.withReader("UTF-8", {
                 props.load(it)
             })
+            if (!props.containsKey("nickname"))
+                props.setProperty("nickname", selectNickname())
             props = new MuWireSettings(props)
             if (props.incompleteLocation == null)
                 props.incompleteLocation = new File(home, "incompletes")
         } else {
             log.info("creating new properties")
             props = new MuWireSettings()
-            props.incompleteLocation = new File(home, "incompletes")
-            props.embeddedRouter = Boolean.parseBoolean(System.getProperties().getProperty("embeddedRouter"))
+            boolean embeddedRouterAvailable = Boolean.parseBoolean(System.getProperties().getProperty("embeddedRouter"))
+            
+            def parent = application.windowManager.findWindow("event-list")
+            Properties i2pProps = new Properties()
+            
+            def params = [:]
+            params['parent'] = parent
+            params['embeddedRouterAvailable'] = embeddedRouterAvailable
+            params['muSettings'] = props
+            params['i2pProps'] = i2pProps
+            def finished = [:]
+            params['finished'] = finished
+            
+            application.mvcGroupManager.createMVCGroup("wizard", params)
+            
+            if (!finished['applied']) {
+                JOptionPane.showMessageDialog(parent, "MuWire will now exit")
+                System.exit(0)
+            }
+            
+            File i2pPropsFile = new File(home, "i2p.properties")
+            i2pPropsFile.withPrintWriter { i2pProps.store(it, "") }     
+            
+            props.embeddedRouter = embeddedRouterAvailable
             props.updateType = System.getProperty("updateType","jar")
-            def nickname
-            while (true) {
-                nickname = JOptionPane.showInputDialog(null,
-                        "Your nickname is displayed when you send search results so other MuWire users can choose to trust you",
-                        "Please choose a nickname", JOptionPane.PLAIN_MESSAGE)
-                if (nickname == null) {
-                    JOptionPane.showMessageDialog(null, "MuWire cannot start without a nickname and will now exit", JOptionPane.PLAIN_MESSAGE)
-                    System.exit(0)
-                }
-                if (nickname.trim().length() == 0) {
-                    JOptionPane.showMessageDialog(null, "Nickname cannot be empty", "Select another nickname",
-                        JOptionPane.WARNING_MESSAGE)
-                    continue
-                }
-                if (nickname.contains("@")) {
-                    JOptionPane.showMessageDialog(null, "Nickname cannot contain @, choose another",
-                        "Select another nickname", JOptionPane.WARNING_MESSAGE)
-                    continue
-                }
-                nickname = nickname.trim()
-                break
-            }
-            props.setNickname(nickname)
-
-
-            def portableDownloads = System.getProperty("portable.downloads")
-            if (portableDownloads != null) {
-                props.downloadLocation = new File(portableDownloads)
-            } else {
-                def chooser = new JFileChooser()
-                chooser.setFileHidingEnabled(false)
-                chooser.setDialogTitle("Select a directory where downloads will be saved")
-                chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY)
-                int rv = chooser.showOpenDialog(null)
-                if (rv != JFileChooser.APPROVE_OPTION) {
-                    JOptionPane.showMessageDialog(null, "MuWire will now exit")
-                    System.exit(0)
-                }
-                props.downloadLocation = chooser.getSelectedFile()
-            }
-
+                   
+            
             propsFile.withPrintWriter("UTF-8", {
                 props.write(it)
             })
@@ -102,23 +89,50 @@ class Ready extends AbstractLifecycleHandler {
         Core core
         try {
             core = new Core(props, home, metadata["application.version"])
+            Runtime.getRuntime().addShutdownHook({
+                core.shutdown()
+            })
+            core.startServices()
+            application.context.put("muwire-settings", props)
+            application.context.put("core",core)
+            application.getPropertyChangeListeners("core").each {
+                it.propertyChange(new PropertyChangeEvent(this, "core", null, core))
+            }
+
+            core.eventBus.publish(new UILoadedEvent())
         } catch (Exception bad) {
             log.log(Level.SEVERE,"couldn't initialize core",bad)
             JOptionPane.showMessageDialog(null, "Couldn't connect to I2P router.  Make sure I2P is running and restart MuWire",
-                "Can't connect to I2P router", JOptionPane.WARNING_MESSAGE)
+                    "Can't connect to I2P router", JOptionPane.WARNING_MESSAGE)
             System.exit(0)
         }
-        Runtime.getRuntime().addShutdownHook({
-            core.shutdown()
-        })
-        core.startServices()
-        application.context.put("muwire-settings", props)
-        application.context.put("core",core)
-        application.getPropertyChangeListeners("core").each {
-            it.propertyChange(new PropertyChangeEvent(this, "core", null, core))
+    }
+    
+    private String selectNickname() {
+        String nickname
+        while (true) {
+            nickname = JOptionPane.showInputDialog(null,
+                    "Your nickname is displayed when you send search results so other MuWire users can choose to trust you",
+                    "Please choose a nickname", JOptionPane.PLAIN_MESSAGE)
+                    if (nickname == null) {
+                        JOptionPane.showMessageDialog(null, "MuWire cannot start without a nickname and will now exit", JOptionPane.PLAIN_MESSAGE)
+                        System.exit(0)
+                    }
+            if (nickname.trim().length() == 0) {
+                JOptionPane.showMessageDialog(null, "Nickname cannot be empty", "Select another nickname",
+                        JOptionPane.WARNING_MESSAGE)
+                continue
+            }
+            if (!DataUtil.isValidName(nickname)) {
+                JOptionPane.showMessageDialog(null, 
+                    "Nickname cannot contain any of ${Constants.INVALID_NICKNAME_CHARS} and must be no longer than ${Constants.MAX_NICKNAME_LENGTH} characters.  Choose another.",
+                        "Select another nickname", JOptionPane.WARNING_MESSAGE)
+                continue
+            }
+            nickname = nickname.trim()
+            break
         }
-
-        core.eventBus.publish(new UILoadedEvent())
+        nickname
     }
 }
 

gui/griffon-app/models/com/muwire/gui/AdvancedSharingModel.groovy

@@ -1,32 +1,49 @@
 package com.muwire.gui
 
+import javax.annotation.Nonnull
 import javax.swing.tree.DefaultMutableTreeNode
 import javax.swing.tree.DefaultTreeModel
 import javax.swing.tree.MutableTreeNode
 
 import com.muwire.core.Core
 import com.muwire.core.files.FileTree
+import com.muwire.core.files.directories.WatchedDirectoryConfigurationEvent
+import com.muwire.core.files.directories.WatchedDirectorySyncEvent
 
 import griffon.core.artifact.GriffonModel
+import griffon.inject.MVCMember
 import griffon.transform.Observable
 import griffon.metadata.ArtifactProviderFor
 
 @ArtifactProviderFor(GriffonModel)
 class AdvancedSharingModel {
+    
+    @MVCMember @Nonnull
+    AdvancedSharingView view
+    
     def watchedDirectories = []
     def treeRoot
     def negativeTree
     
     Core core
     
+    @Observable boolean syncActionEnabled
+    
     void mvcGroupInit(Map<String,String> args) {
-        watchedDirectories.addAll(core.muOptions.watchedDirectories)
+        watchedDirectories.addAll(core.watchedDirectoryManager.watchedDirs.values())
+        core.eventBus.register(WatchedDirectorySyncEvent.class, this)
+        core.eventBus.register(WatchedDirectoryConfigurationEvent.class, this)
         
         treeRoot = new DefaultMutableTreeNode()
         negativeTree = new DefaultTreeModel(treeRoot)
         copyTree(treeRoot, core.fileManager.negativeTree.root)
     }
     
+    void mvcGroupDestroy() {
+        core.eventBus.unregister(WatchedDirectorySyncEvent.class, this)
+        core.eventBus.unregister(WatchedDirectoryConfigurationEvent.class, this)
+    }
+    
     private void copyTree(DefaultMutableTreeNode jtreeNode, FileTree.TreeNode fileTreeNode) {
         jtreeNode.setUserObject(fileTreeNode.file?.getName())
         fileTreeNode.children.each { 
@@ -36,4 +53,16 @@ class AdvancedSharingModel {
         }
     }
     
+    void onWatchedDirectorySyncEvent(WatchedDirectorySyncEvent e) {
+        runInsideUIAsync {
+            view.watchedDirsTable.model.fireTableDataChanged()
+        }
+    }
+    
+    void onWatchedDirectoryConfigurationEvent(WatchedDirectoryConfigurationEvent e) {
+        runInsideUIAsync {
+            view.watchedDirsTable.model.fireTableDataChanged()
+        }
+    }
+    
 }