loveisgrief/muwire
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove trailing spaces

Browse Source
This commit is contained in:
Zlatin Balevsky
2019-07-05 16:24:19 +01:00
parent 94bb7022eb
commit 29cdbf018c
126 changed files with 1952 additions and 1541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy
View File

@ -12,30 +12,30 @@ class Crawler {
final def pinger
final def hostPool
final int parallel
final Map<Destination, Host> inFlight = new HashMap<>()
UUID currentUUID
Crawler(pinger, hostPool, int parallel) {
this.pinger = pinger
this.hostPool = hostPool
this.parallel = parallel
}
synchronized def handleCrawlerPong(pong, Destination source) {
if (!inFlight.containsKey(source)) {
log.info("response from host that hasn't been crawled")
return
}
Host host = inFlight.remove(source)
if (pong.uuid == null || pong.leafSlots == null || pong.peerSlots == null || pong.peers == null) {
hostPool.fail(host)
log.info("invalid crawler pong")
return
}
UUID uuid;
try {
uuid = UUID.fromString(pong.uuid)
@ -43,16 +43,16 @@ class Crawler {
hostPool.fail(host)
return
}
if (!uuid.equals(currentUUID)) {
log.info("uuid mismatch")
hostPool.fail(host)
return
}
host.leafSlots = parseBoolean(pong.leafSlots)
host.peerSlots = parseBoolean(pong.peerSlots)
def peers
try {
peers = pong.peers.stream().map({b64 -> new Destination(b64)}).collect(Collectors.toSet())
@ -69,18 +69,18 @@ class Crawler {
}
hostPool.verify(host)
}
private static boolean parseBoolean(value) {
return Boolean.parseBoolean(value.toString())
}
synchronized def startCrawl() {
if (!inFlight.isEmpty()) {
inFlight.values().each { hostPool.fail(it) }
inFlight.clear()
}
currentUUID = UUID.randomUUID()
hostPool.getUnverified(parallel).each {
hostPool.getUnverified(parallel).each {
inFlight.put(it.destination, it)
pinger.ping(it, currentUUID)
}

4
host-cache/src/main/groovy/com/muwire/hostcache/Host.groovy
View File

@ -7,11 +7,11 @@ class Host {
boolean leafSlots
boolean peerSlots
int verificationFailures
public int hashCode() {
return destination.hashCode()
}
public boolean equals(other) {
return destination.equals(other.destination)
}

38
host-cache/src/main/groovy/com/muwire/hostcache/HostCache.groovy
View File

@ -23,23 +23,23 @@ public class HostCache {
println "This will likely not run on windows"
System.exit(1)
}
def home = System.getProperty("user.home") + "/.MuWireHostCache"
home = new File(home)
if (home.exists() && !home.isDirectory()) {
println "${home} exists but not a directory? Delete it or make it a directory"
System.exit(1)
}
if (!home.exists()) {
home.mkdir()
}
def keyfile = new File(home, "key.dat")
def i2pClientFactory = new I2PClientFactory()
def i2pClient = i2pClientFactory.createClient()
def myDest
def session
if (!keyfile.exists()) {
@ -49,39 +49,39 @@ public class HostCache {
println "No key.dat file was found, so creating a new destination."
println "This is the destination you want to give out for your new HostCache"
println myDest.toBase64()
}
}
def props = System.getProperties().clone()
props.putAt("inbound.nickname", "MuWire HostCache")
session = i2pClient.createSession(new FileInputStream(keyfile), props)
myDest = session.getMyDestination()
// initialize hostpool and crawler
HostPool hostPool = new HostPool(3, 60 * 60 * 1000)
Pinger pinger = new Pinger(session)
Crawler crawler = new Crawler(pinger, hostPool, 5)
Timer timer = new Timer("timer", true)
timer.schedule({hostPool.age()} as TimerTask, 1000,1000)
timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000)
File verified = new File("verified.json")
File unverified = new File("unverified.json")
timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000)
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),
I2PSession.PROTO_DATAGRAM, I2PSession.PORT_ANY)
session.connect()
log.info("connected, going to sleep")
Thread.sleep(Integer.MAX_VALUE)
}
static class Listener implements I2PSessionMuxedListener {
final def json = new JsonSlurper()
def hostPool
int toReturn
def crawler
void reportAbuse(I2PSession sesison, int severity) {}
void disconnected(I2PSession session) {
log.severe("session disconnected, exiting")
@ -103,7 +103,7 @@ public class HostCache {
dissector.loadI2PDatagram(payload)
def sender = dissector.getSender()
def b32 = sender.toBase32()
payload = dissector.getPayload()
payload = json.parse(payload)
if (payload.type == null) {
@ -112,7 +112,7 @@ public class HostCache {
}
switch(payload.type) {
case "Ping" :
log.info("ping from $b32")
log.info("ping from $b32")
if (payload.leaf == null) {
log.warning("ping didn't specify if leaf from $b32")
return
@ -131,16 +131,16 @@ public class HostCache {
}
} catch (Exception dfe) {
log.log(Level.WARNING,"invalid datagram", dfe)
}
}
}
void messageAvailable(I2PSession session, int msgId, long size) {
}
def respond(session, destination, ping) {
def pongs = hostPool.getVerified(toReturn, ping.leaf)
pongs = pongs.stream().map({ x -> x.destination.toBase64() }).collect(Collectors.toList())
def pong = [type:"Pong", version: 1, pongs: pongs]
pong = JsonOutput.toJson(pong)
def maker = new I2PDatagramMaker(session)

28
host-cache/src/main/groovy/com/muwire/hostcache/HostPool.groovy
View File

@ -5,19 +5,19 @@ import java.util.stream.Collectors
import groovy.json.JsonOutput
class HostPool {
final def maxFailures
final def maxAge
def verified = new HashMap()
def unverified = new HashMap()
HostPool() {}
HostPool(maxFailures, maxAge) {
this.maxAge = maxAge
this.maxFailures = maxFailures
}
synchronized def getVerified(int max, boolean leaf) {
if (verified.isEmpty()) {
return Collections.emptyList()
@ -27,13 +27,13 @@ class HostPool {
return asList[0..Math.min(max, asList.size()) -1]
}
synchronized def addUnverified(host) {
if (!verified.containsKey(host.destination)) {
unverified.put(host.destination, host)
}
}
synchronized def getUnverified(int max) {
if (unverified.isEmpty()) {
return Collections.emptyList()
@ -42,7 +42,7 @@ class HostPool {
Collections.shuffle(asList)
return asList[0..(Math.min(max, asList.size())-1)]
}
synchronized def verify(host) {
if (!unverified.remove(host.destination))
throw new IllegalArgumentException()
@ -50,13 +50,13 @@ class HostPool {
host.verificationFailures = 0
verified.put(host.destination, host)
}
synchronized def fail(host) {
if (!unverified.containsKey(host.destination))
return
host.verificationFailures++
}
synchronized def age() {
final long now = System.currentTimeMillis()
for (Iterator iter = verified.keySet().iterator(); iter.hasNext();) {
@ -67,7 +67,7 @@ class HostPool {
unverified.put(host.destination, host)
}
}
for (Iterator iter = unverified.keySet().iterator(); iter.hasNext();) {
def destination = iter.next()
def host = unverified.get(destination)
@ -76,16 +76,16 @@ class HostPool {
}
}
}
synchronized void serialize(File verifiedFile, File unverifiedFile) {
write(verifiedFile, verified.values())
write(unverifiedFile, unverified.values())
}
private void write(File target, Collection hosts) {
JsonOutput jsonOutput = new JsonOutput()
target.withPrintWriter { writer ->
hosts.each {
hosts.each {
def json = [:]
json.destination = it.destination.toBase64()
json.verifyTime = it.verifyTime
@ -94,7 +94,7 @@ class HostPool {
json.verificationFailures = it.verificationFailures
def str = jsonOutput.toJson(json)
writer.println(str)
}
}
}
}
}

4
host-cache/src/main/groovy/com/muwire/hostcache/Pinger.groovy
View File

@ -5,12 +5,12 @@ import net.i2p.client.I2PSession
import net.i2p.client.datagram.I2PDatagramMaker
class Pinger {
final def session
Pinger(session) {
this.session = session
}
def ping(host, uuid) {
def maker = new I2PDatagramMaker(session)
def payload = new HashMap()

60
host-cache/src/test/groovy/com/muwire/hostcache/CrawlerTest.groovy
View File

@ -12,45 +12,45 @@ class CrawlerTest {
def pingerMock
def pinger
def hostPoolMock
def hostPool
def crawler
Destinations destinations = new Destinations()
final Host host = new Host(destination: new Destination())
final Host host1 = new Host(destination: destinations.dest1)
final Host host2 = new Host(destination: destinations.dest2)
final int parallel = 5
@Before
void before() {
pingerMock = new MockFor(Pinger)
hostPoolMock = new MockFor(HostPool)
}
@After
void after() {
hostPoolMock.verify hostPool
pingerMock.verify pinger
}
private def initCrawler() {
pinger = pingerMock.proxyInstance()
hostPool = hostPoolMock.proxyInstance()
crawler = new Crawler(pinger, hostPool, parallel)
}
@Test
void testBadJson() {
initCrawler()
def unpingedHost = new Host(destination : new Destination())
crawler.handleCrawlerPong(null, new Destination())
}
@Test
void testStartCrawl() {
hostPoolMock.demand.getUnverified { n ->
@ -58,12 +58,12 @@ class CrawlerTest {
[host]
}
pingerMock.demand.ping { h,uuid -> assert h == host }
initCrawler()
crawler.startCrawl()
}
@Test
void testFailsUnanswered() {
hostPoolMock.demand.getUnverified {n -> [host]}
@ -71,40 +71,40 @@ class CrawlerTest {
hostPoolMock.demand.getUnverified {n -> [:]}
pingerMock.demand.ping {h,uuid -> }
initCrawler()
crawler.startCrawl()
crawler.startCrawl()
}
@Test
void testVerifiesAnswered() {
def currentUUID
hostPoolMock.demand.getUnverified { n -> [host1] }
hostPoolMock.demand.verify { h -> assert h == host1 }
pingerMock.demand.ping { h, uuid -> currentUUID = uuid }
initCrawler()
crawler.startCrawl()
def pong = [uuid : currentUUID.toString(), leafSlots : "false", peerSlots: "false", peers: [] ]
crawler.handleCrawlerPong(pong, host1.destination)
}
@Test
void testWrongSourceIgnored() {
def currentUUID
hostPoolMock.demand.getUnverified { n -> [host1] }
pingerMock.demand.ping { h, uuid -> currentUUID = uuid }
initCrawler()
crawler.startCrawl()
def pong = [uuid : currentUUID.toString(), leafSlots : "false", peerSlots: "false", peers: [] ]
crawler.handleCrawlerPong(pong, host2.destination)
}
@Test
void testHost1CarriesHost2() {
def currentUUID
@ -112,26 +112,26 @@ class CrawlerTest {
hostPoolMock.demand.addUnverified { h -> assert h == host2 }
hostPoolMock.demand.verify { h -> assert h == host1 }
pingerMock.demand.ping { h, uuid -> currentUUID = uuid }
initCrawler()
crawler.startCrawl()
def pong = [uuid : currentUUID.toString(), leafSlots : "false", peerSlots: "false", peers: [destinations.dest2.toBase64()] ]
crawler.handleCrawlerPong(pong, host1.destination)
}
@Test
void testWrongUUID() {
def currentUUID
hostPoolMock.demand.getUnverified { n -> [host1] }
hostPoolMock.demand.fail { h -> assert h == host1 }
pingerMock.demand.ping { h, uuid -> currentUUID = uuid }
initCrawler()
crawler.startCrawl()
def pong = [uuid : UUID.randomUUID().toString(), leafSlots : "false", peerSlots: "false", peers: [] ]
crawler.handleCrawlerPong(pong, host1.destination)
}

76
host-cache/src/test/groovy/com/muwire/hostcache/HostPoolTest.groovy
View File

@ -3,65 +3,65 @@ package com.muwire.hostcache
import org.junit.*
class HostPoolTest {
def hp
Host freeLeafs, freePeers, freeBoth, freeNone
@Before
void before() {
hp = new HostPool(3, 10)
freeLeafs = new Host()
freeLeafs.destination = "freeLeafs"
freeLeafs.leafSlots = true
freePeers = new Host()
freePeers.destination = "freePeers"
freePeers.peerSlots = true
freeBoth = new Host()
freeBoth.destination = "freeBoth"
freeBoth.leafSlots = true
freeBoth.peerSlots = true
freeNone = new Host()
freeNone.destination = "freeNone"
}
def addAllUnverified() {
hp.addUnverified(freeLeafs)
hp.addUnverified(freePeers)
hp.addUnverified(freeBoth)
hp.addUnverified(freeNone)
}
def verifyAll() {
hp.verify(freeBoth)
hp.verify(freeLeafs)
hp.verify(freePeers)
hp.verify(freeNone)
}
@Test
void testNoVerified() {
void testNoVerified() {
assert hp.getVerified(1, true).isEmpty()
assert hp.getVerified(1, false).isEmpty()
addAllUnverified()
addAllUnverified()
assert hp.getVerified(1, true).isEmpty()
assert hp.getVerified(1, false).isEmpty()
}
@Test
void testOneVerified() {
addAllUnverified()
hp.verify(freeBoth)
assert hp.getVerified(10, true).contains(freeBoth)
assert hp.getVerified(10, false).contains(freeBoth)
}
@Test
void testFilterByType() {
addAllUnverified()
@ -70,13 +70,13 @@ class HostPoolTest {
assert verifiedLeafSlots.size() == 2
assert verifiedLeafSlots.contains(freeBoth)
assert verifiedLeafSlots.contains(freeLeafs)
def verifiedPeerSlots = hp.getVerified(10, false)
assert verifiedPeerSlots.size() == 2
assert verifiedPeerSlots.contains(freeBoth)
assert verifiedPeerSlots.contains(freePeers)
}
@Test
void getFewerThanAvailable() {
addAllUnverified()
@ -85,88 +85,88 @@ class HostPoolTest {
assert verifiedLeafSlots.size() == 1
assert verifiedLeafSlots.contains(freeBoth) || verifiedLeafSlots.contains(freeLeafs)
}
@Test
void getUnverified() {
assert hp.getUnverified().isEmpty()
addAllUnverified()
def allUnverified = hp.getUnverified(10)
assert allUnverified.size() == 4
assert allUnverified.contains(freeLeafs)
assert allUnverified.contains(freePeers)
assert allUnverified.contains(freeBoth)
assert allUnverified.contains(freeNone)
def twoUnverified = hp.getUnverified(2)
assert twoUnverified.size() == 2
def oneUnverified = hp.getUnverified(1)
assert oneUnverified.size() == 1
assert oneUnverified.contains(freeLeafs) || oneUnverified.contains(freePeers) ||
oneUnverified.contains(freeBoth) || oneUnverified.contains(freeNone)
}
@Test
void testFailHost() {
hp.addUnverified(freeBoth)
assert hp.getUnverified(10).size() == 1
hp.fail(freeBoth)
hp.age()
assert hp.getUnverified(10).size() == 1
hp.fail(freeBoth)
hp.age()
assert hp.getUnverified(10).size() == 1
hp.fail(freeBoth)
hp.age()
assert hp.getUnverified(10).isEmpty()
assert hp.getVerified(10, true).isEmpty()
}
@Test
void verifyResetsFailures() {
hp.addUnverified(freeBoth)
assert hp.getUnverified(10).size() == 1
hp.fail(freeBoth)
hp.age()
assert hp.getUnverified(10).size() == 1
hp.fail(freeBoth)
hp.age()
assert hp.getUnverified(10).size() == 1
hp.verify(freeBoth)
hp.fail(freeBoth)
hp.age()
assert hp.getUnverified(10).size() == 1
hp.verify(freeBoth)
hp.age()
assert hp.getUnverified(10).isEmpty()
assert hp.getVerified(10, true).size() == 1
}
@Test
void ageHost() {
hp.addUnverified(freeBoth)
hp.verify(freeBoth)
hp.age()
assert hp.getVerified(10,true).size() == 1
assert hp.getUnverified(10).isEmpty()
Thread.sleep(20)
hp.age()
assert hp.getVerified(10,true).isEmpty()
assert hp.getUnverified(10).size() == 1
}
@Test
void doNotAddIfVerified() {
hp.addUnverified(freeBoth)
hp.verify(freeBoth)
assert hp.getUnverified(1).isEmpty()
hp.addUnverified(freeBoth)
assert hp.getUnverified(1).isEmpty()
}