From 656ebbe89947f0316aa1e6ea5b802c91b6a309f2 Mon Sep 17 00:00:00 2001 From: zzz Date: Thu, 25 Apr 2019 13:34:54 +0000 Subject: [PATCH] newsxml server by psi, public domain --- newsxml-flask/README.txt | 47 ++++++ newsxml-flask/app.py | 103 ++++++++++++ newsxml-flask/run.sh | 6 + newsxml-flask/settings.json | 5 + newsxml-flask/stats.py | 259 +++++++++++++++++++++++++++++ newsxml-flask/templates/index.html | 12 ++ 6 files changed, 432 insertions(+) create mode 100644 newsxml-flask/README.txt create mode 100644 newsxml-flask/app.py create mode 100755 newsxml-flask/run.sh create mode 100644 newsxml-flask/settings.json create mode 100644 newsxml-flask/stats.py create mode 100644 newsxml-flask/templates/index.html diff --git a/newsxml-flask/README.txt b/newsxml-flask/README.txt new file mode 100644 index 0000000..4c3fb0e --- /dev/null +++ b/newsxml-flask/README.txt @@ -0,0 +1,47 @@ +i2p.newsxml server. put su3 files into ./static and serve them + +license: public domain + +===== + +dependancies: + +python 2.7 or 3.4+ + +flask (required): + + # pip install flask + +graphs: (optional) + + # pip install pygal + +persistant statistics storage: (optional) + + # apt install redis-server python-redis + + +===== + +setup: + +it binds an http server to 127.0.0.1:9696 by default + +run so it is externally accessable via http://your.server.tld/news/ and uses the internal port 1234 + + $ ./run.sh http://your.server.tld/news/ 1234 + +you need to reverse proxy this or use another i2p destination for news + +if you do another destination run: + + $ ./run.sh http://your.b32.i2p/ 1234 + +then point an http SERVER tunnel (not http bidir) to 127.0.0.1 port 1234 + + +===== + +questions: + +ask psi on irc2p diff --git a/newsxml-flask/app.py b/newsxml-flask/app.py new file mode 100644 index 0000000..8f48e11 --- /dev/null +++ b/newsxml-flask/app.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# +# -*- coding: utf-8 -*- +# +import flask +import logging +import sys +import os + +app = flask.Flask(__name__) + +def hit(x): + pass +try: + import stats + hit = stats.engine.hit +except ImportError: + stats = None + +# this is the root path for news.xml server, must end it / +# i.e. http://news.psi.i2p/news/ +# defaults to / +ROOT='/' +port=9696 +if len(sys.argv) > 1: + ROOT=sys.argv[1] + if len(sys.argv) > 2: + port = int(sys.argv[2]) + +@app.route('/') +def index(): + """ + serve news stats page + """ + return flask.render_template('index.html',root=ROOT) + +def has_lang(lang): + """ + :return True if we have news for a language: + """ + if '.' in lang or '/' in lang: + return False + return os.path.exists(os.path.join(app.static_folder, 'news_{}.su3'.format(lang))) + +@app.route('/news.su3') +def news_su3(): + """ + serve news.su3 + """ + lang = flask.request.args.get('lang', 'en') + lang = lang.split('_')[0] + hit(lang) + fname = 'news.su3' + if has_lang(lang): + fname = 'news_{}.su3'.format(lang) + + return flask.send_file(os.path.join(app.static_folder, fname)) + +@app.route('/netsize.svg') +def netsize_svg(): + """ + generate and serve network size svg + """ + if stats: + args = flask.request.args + try: + window = int(args['window']) + tslice = int(args['tslice']) + mult = int(args['mult']) + resp = flask.Response(stats.engine.netsize(tslice, window, mult)) + resp.mimetype = 'image/svg+xml' + return resp + except Exception as e: + print (e) + flask.abort(503) + # we don't have stats to show, stats module not imported + flask.abort(404) + + +@app.route('/requests.svg') +def requests_svg(): + """ + generate and serve requests per interval graph + """ + args = flask.request.args + if stats: + try: + window = int(args['window']) + tslice = int(args['tslice']) + mult = int(args['mult']) + resp = flask.Response(stats.engine.requests(tslice, window, mult)) + resp.mimetype = 'image/svg+xml' + return resp + except Exception as e: + print (e) + flask.abort(503) + flask.abort(404) + + +if __name__ == '__main__': + # run it + logging.basicConfig(level=logging.INFO) + app.run('127.0.0.1', port) diff --git a/newsxml-flask/run.sh b/newsxml-flask/run.sh new file mode 100755 index 0000000..04bd31f --- /dev/null +++ b/newsxml-flask/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# +# run newsxml server +# +cd $(dirname $0) +PYTHONPATH=/usr/local/lib/python2.7/dist-packages python app.py &> newsxml.log & diff --git a/newsxml-flask/settings.json b/newsxml-flask/settings.json new file mode 100644 index 0000000..35e0a8a --- /dev/null +++ b/newsxml-flask/settings.json @@ -0,0 +1,5 @@ +{ + "slice" : 480, + "mult" : 3, + "winlen" : 30 +} diff --git a/newsxml-flask/stats.py b/newsxml-flask/stats.py new file mode 100644 index 0000000..d2fc39b --- /dev/null +++ b/newsxml-flask/stats.py @@ -0,0 +1,259 @@ +# +# -*- coding: utf-8 -*- +# +import datetime +import json +import logging +import operator +import os +import time + +# try importing redis redis +try: + import redis +except ImportError: + print("redis not available, fall back to volatile stats backend") + redis = None + +# try importing pygal +try: + import pygal +except ImportError: + print("pygal not available, fall back to text based stats") + pygal = None +pygal =None + +__doc__ = """ +statistics backend optionally using redis +""" + + +class RedisDB: + """ + redis based backend for storing stats + """ + def __init__(self): + self._redis = redis.Redis() + self.exists = self._redis.exists + self.get = self._redis.get + self.set = self._redis.set + +class DictDB: + """ + volatile dictionary based database backend for storing stats in memory + """ + def __init__(self): + self._d = dict() + + def get(self, k): + if self.exists(k): + return self._d[k] + + def set(self, k, v): + self._d[k] = v + + def exists(self, k): + return k in self._d + + +class Grapher: + """ + generic grapher that does nothing + """ + + def collect(self, data_sorted, multiplier, calc_netsize): + """ + do the magic calculations + yields (x, netsize_y, rph_y) + """ + total = 0 + hours = 0 + req_s = [] + netsize_s = [] + window = [] + for hour, val in data_sorted: + years = hour / ( 365 * 24 ) + days = ( hour - years * 365 * 24 ) / 24 + hours = hour - ( ( years * 365 * 24 ) + ( days * 24 ) ) + hour = datetime.datetime.strptime('%0.4d_%0.3d_%0.2d' % (years, days, hours), '%Y_%j_%H') + if val > 0: + total += val + hours += 1 + per_hour = float(total) / hours + window.append(val) + while len(window) > window_len: + window.pop(0) + mean = sum(window) / len(window) + netsize = int(calc_netsize(mean, multiplier)) + yield (hour, netsize, val) + + def generate(self, data_sorted, multiplier, calc_netsize): + """ + :param data_sorted: sorted list of (hour, hitcount) tuple + :param multiplier: multiplier to use on graph Y axis + :param calc_netsize: function that calculates the network size given a mean value and multiplier + :return (netsize, requests) graph tuple: + """ + +class SVGText: + """ + svg hold text + """ + def __init__(self, data='undefined'): + self.data = data + + def render(self): + return """ + + fallback svg + + + {} + + """.format(self.data) + +class TextGrapher(Grapher): + """ + generates svg manually that look like ass + """ + + def generate(self, data_sorted, multiplier, calc_netsize): + nsize = 0 + rph = 0 + t = 0 + for hour, netsize, reqs in self.collect(data_sorted, multiplier, calc_netsize): + t += 1 + nsize += netsize + rpy += reqs + if t: + nsize /= t + rph /= t + return SVGText("MEAN NETSIZE: {} routers".format(nsize)), SVGText("MEAN REQUETS: {} req/hour".format(rph)) + +class PygalGrapher(Grapher): + """ + generates svg graphs using pygal + """ + + def generate(self, data_sorted, multiplier, calc_netsize): + + _netsize_graph = pygal.DateY(show_dots=False,x_label_rotation=20) + _requests_graph = pygal.DateY(show_dots=False,x_label_rotation=20) + + _netsize_graph.title = 'Est. Network Size (multiplier: %d)' % multiplier + _requests_graph.title = 'Requests Per Hour' + + netsize_s, req_s = list(), list() + for hour, netsize, reqs in self.collect(data_sorted, multiplier, calc_netsize): + netsize_s.append((hour, netsize)) + req_s.append((hour, reqs)) + + _netsize_graph.add('Routers', netsize_s) + _requests_graph.add('news.xml Requests', req_s) + return _netsize_graph, _requests_graph + + +class StatsEngine: + """ + Stats engine for news.xml + """ + + _log = logging.getLogger('StatsEngine') + + def __init__(self): + self._cfg_fname = 'settings.json' + if redis: + self._db = RedisDB() + try: + self._db.exists('nothing') + except: + self._log.warn("failed to connect to redis, falling back to volatile stats backend") + self._db = DictDB() + else: + self._db = DictDB() + if pygal: + self._graphs = PygalGrapher() + else: + self._graphs = TextGrapher() + + self._last_hour = self.get_hour() + + def _config_str(self, name): + with open(self._cfg_fname) as f: + return str(json.load(f)[name]) + + def _config_int(self, name): + with open(self._cfg_fname) as f: + return int(json.load(f)[name]) + + def multiplier(self): + return self._config_int('mult') + + def tslice(self): + return self._config_int('slice') + + def window_len(self): + return self._config_int('winlen') + + @staticmethod + def get_hour(): + """ + get the current our as an int + """ + dt = datetime.datetime.utcnow() + return dt.hour + (int(dt.strftime('%j')) * 24 ) + ( dt.year * 24 * 365 ) + + def calc_netsize(self, per_hour, mult): + return float(per_hour) * 24 / 1.5 * mult + + @staticmethod + def _hour_key(hour): + return 'newsxml.hit.{}'.format(hour) + + def hit(self, lang=None): + """ + record a request + """ + hour = self.get_hour() + keyname = self._hour_key(hour) + if not self._db.exists(keyname): + self._db.set(keyname, '0') + val = self._db.get(keyname) + self._db.set(keyname, str(int(val) + 1)) + + def _load_data(self, hours): + """ + load hit data + """ + hour = self.get_hour() + data = list() + while hours > 0: + keyname = self._hour_key(hour) + val = self._db.get(keyname) + if val: + data.append((hour, int(val))) + hour -= 1 + hours -= 1 + return data + + def regen_graphs(self, tslice, window_len, mult): + data = self._load_data(tslice) + data_sorted = sorted(data, key=operator.itemgetter(0)) + if len(data_sorted) > tslice: + data_sorted = data_sorted[-tslice:] + self._netsize_graph, self._requests_graph = self._graphs.generate(data_sorted, self.multiplier(), self.calc_netsize) + + + + def netsize(self, tslice, window, mult): + #if not hasattr(self,'_netsize_graph'): + self.regen_graphs(tslice, window, mult) + return self._netsize_graph.render() + + def requests(self, tslice, window, mult): + #if not hasattr(self,'_requests_graph'): + self.regen_graphs(tslice, window, mult) + return self._requests_graph.render() + + +engine = StatsEngine() diff --git a/newsxml-flask/templates/index.html b/newsxml-flask/templates/index.html new file mode 100644 index 0000000..4ef1c74 --- /dev/null +++ b/newsxml-flask/templates/index.html @@ -0,0 +1,12 @@ + + + news.xml stats + + + + + + + + +