Merge branch 'py3' into plugins

2019-07-12 01:28:32 +07:00 · 2019-07-12 01:28:32 +07:00 · 735061b79d
commit 735061b79d
parent aa6d7a468d a2cb1615b3
15 changed files with 212 additions and 153 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -15,7 +15,6 @@ before_install:
  # - docker build -t zeronet .
  # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet
 install:
- - if [[ $TRAVIS_PYTHON_VERSION == 3.8-dev ]]; then pip install --upgrade setuptools cffi 'cython>=0.28' git+git://github.com/gevent/gevent.git#egg=gevent; fi
 - pip install --upgrade -r requirements.txt
 - pip list
 before_script:
--- a/README.md
+++ b/README.md
@ -1,5 +1,3 @@
-__Warning: Development test version, do not use on live data__
-
 # ZeroNet [![Build Status](https://travis-ci.org/HelloZeroNet/ZeroNet.svg?branch=master)](https://travis-ci.org/HelloZeroNet/ZeroNet) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://zeronet.io/docs/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://zeronet.io/docs/help_zeronet/donate/)

 Decentralized websites using Bitcoin crypto and the BitTorrent network - https://zeronet.io
@ -63,19 +61,18 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/

 ## How to join

-### Install from package for your distribution
+### Windows

-* Arch Linux: [zeronet](https://aur.archlinux.org/zeronet.git), [zeronet-git](https://aur.archlinux.org/zeronet-git.git)
-* Gentoo:  [emerge repository](https://github.com/leycec/raiagent)
-* FreeBSD: zeronet
-* Whonix: [instructions](https://www.whonix.org/wiki/ZeroNet)
+ - Download [ZeroNet-py3-win64.zip](https://github.com/HelloZeroNet/ZeroNet-win/archive/dist-win64/ZeroNet-py3-win64.zip) (18MB)
+ - Unpack anywhere
+ - Run `ZeroNet.exe`

-### Install from source
+### Other platforms: Install from source

 Fetch and extract the source:

-    wget https://github.com/HelloZeroNet/ZeroNet/archive/py3.tar.gz
-    tar xvpfz py3.tar.gz
+    wget https://github.com/HelloZeroNet/ZeroNet/archive/py3/ZeroNet-py3.tar.gz
+    tar xvpfz ZeroNet-py3.tar.gz
    cd ZeroNet-py3

 Install Python module dependencies either:
--- a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py
+++ b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py
@ -0,0 +1,150 @@
+import time
+import urllib.request
+import struct
+import socket
+
+import bencode
+from lib.subtl.subtl import UdpTrackerClient
+import socks
+import sockshandler
+import gevent
+
+from Plugin import PluginManager
+from Config import config
+from Debug import Debug
+from util import helper
+
+
+# We can only import plugin host clases after the plugins are loaded
+@PluginManager.afterLoad
+def importHostClasses():
+    global Peer, AnnounceError
+    from Peer import Peer
+    from Site.SiteAnnouncer import AnnounceError
+
+
+@PluginManager.registerTo("SiteAnnouncer")
+class SiteAnnouncerPlugin(object):
+    def getSupportedTrackers(self):
+        trackers = super(SiteAnnouncerPlugin, self).getSupportedTrackers()
+        if config.disable_udp or config.trackers_proxy != "disable":
+            trackers = [tracker for tracker in trackers if not tracker.startswith("udp://")]
+
+        return trackers
+
+    def getTrackerHandler(self, protocol):
+        if protocol == "udp":
+            handler = self.announceTrackerUdp
+        elif protocol == "http":
+            handler = self.announceTrackerHttp
+        elif protocol == "https":
+            handler = self.announceTrackerHttps
+        else:
+            handler = super(SiteAnnouncerPlugin, self).getTrackerHandler(protocol)
+        return handler
+
+    def announceTrackerUdp(self, tracker_address, mode="start", num_want=10):
+        s = time.time()
+        if config.disable_udp:
+            raise AnnounceError("Udp disabled by config")
+        if config.trackers_proxy != "disable":
+            raise AnnounceError("Udp trackers not available with proxies")
+
+        ip, port = tracker_address.split("/")[0].split(":")
+        tracker = UdpTrackerClient(ip, int(port))
+        if helper.getIpType(ip) in self.getOpenedServiceTypes():
+            tracker.peer_port = self.fileserver_port
+        else:
+            tracker.peer_port = 0
+        tracker.connect()
+        if not tracker.poll_once():
+            raise AnnounceError("Could not connect")
+        tracker.announce(info_hash=self.site.address_sha1, num_want=num_want, left=431102370)
+        back = tracker.poll_once()
+        if not back:
+            raise AnnounceError("No response after %.0fs" % (time.time() - s))
+        elif type(back) is dict and "response" in back:
+            peers = back["response"]["peers"]
+        else:
+            raise AnnounceError("Invalid response: %r" % back)
+
+        return peers
+
+    def httpRequest(self, url):
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
+            'Accept-Encoding': 'none',
+            'Accept-Language': 'en-US,en;q=0.8',
+            'Connection': 'keep-alive'
+        }
+
+        req = urllib.request.Request(url, headers=headers)
+
+        if config.trackers_proxy == "tor":
+            tor_manager = self.site.connection_server.tor_manager
+            handler = sockshandler.SocksiPyHandler(socks.SOCKS5, tor_manager.proxy_ip, tor_manager.proxy_port)
+            opener = urllib.request.build_opener(handler)
+            return opener.open(req, timeout=50)
+        elif config.trackers_proxy == "disable":
+            return urllib.request.urlopen(req, timeout=25)
+        else:
+            proxy_ip, proxy_port = config.trackers_proxy.split(":")
+            handler = sockshandler.SocksiPyHandler(socks.SOCKS5, proxy_ip, int(proxy_port))
+            opener = urllib.request.build_opener(handler)
+            return opener.open(req, timeout=50)
+
+    def announceTrackerHttps(self, *args, **kwargs):
+        kwargs["protocol"] = "https"
+        return self.announceTrackerHttp(*args, **kwargs)
+
+    def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protocol="http"):
+        tracker_ip, tracker_port = tracker_address.rsplit(":", 1)
+        if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes():
+            port = self.fileserver_port
+        else:
+            port = 1
+        params = {
+            'info_hash': self.site.address_sha1,
+            'peer_id': self.peer_id, 'port': port,
+            'uploaded': 0, 'downloaded': 0, 'left': 431102370, 'compact': 1, 'numwant': num_want,
+            'event': 'started'
+        }
+
+        url = protocol + "://" + tracker_address + "?" + urllib.parse.urlencode(params)
+
+        s = time.time()
+        response = None
+        # Load url
+        if config.tor == "always" or config.trackers_proxy != "disable":
+            timeout = 60
+        else:
+            timeout = 30
+
+        with gevent.Timeout(timeout, False):  # Make sure of timeout
+            req = self.httpRequest(url)
+            response = req.read()
+            req.close()
+            req = None
+
+        if not response:
+            raise AnnounceError("No response after %.0fs" % (time.time() - s))
+
+        # Decode peers
+        try:
+            peer_data = bencode.decode(response)["peers"]
+            if type(peer_data) is not bytes:
+                peer_data = peer_data.encode()
+            response = None
+            peer_count = int(len(peer_data) / 6)
+            peers = []
+            for peer_offset in range(peer_count):
+                off = 6 * peer_offset
+                peer = peer_data[off:off + 6]
+                addr, port = struct.unpack('!LH', peer)
+                peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port})
+        except Exception as err:
+            raise AnnounceError("Invalid response: %r (%s)" % (response, Debug.formatException(err)))
+
+        return peers
--- a/plugins/AnnounceBitTorrent/init.py
+++ b/plugins/AnnounceBitTorrent/init.py
@ -0,0 +1 @@
+from . import AnnounceBitTorrentPlugin
--- a/plugins/OptionalManager/UiWebsocketPlugin.py
+++ b/plugins/OptionalManager/UiWebsocketPlugin.py
@ -105,7 +105,7 @@ class UiWebsocketPlugin(object):

    # Optional file functions

-    def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC", limit=10, filter="downloaded"):
+    def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC", limit=10, filter="downloaded", filter_inner_path=None):
        if not address:
            address = self.site.address

@ -139,6 +139,8 @@ class UiWebsocketPlugin(object):
            wheres_raw.append("(is_downloaded = 1 OR is_pinned = 1)")
        if "pinned" in filter:
            wheres["is_pinned"] = 1
+        if filter_inner_path:
+            wheres["inner_path__like"] = filter_inner_path

        if address == "all":
            join = "LEFT JOIN site USING (site_id)"
--- a/plugins/UiConfig/languages/zh.json
+++ b/plugins/UiConfig/languages/zh.json
@ -4,6 +4,8 @@
 	"Open web browser on ZeroNet startup": "ZeroNet启动时,打开浏览器",

 	"Network": "网络",
+	"Offline mode": "离线模式",
+	"Disable network communication.": "关闭网络通信.",
 	"File server network": "文件服务器网络",
 	"Accept incoming peers using IPv4 or IPv6 address. (default: dual)": "使用IPv4或IPv6地址接受传入的节点请求. (默认:双重)",
 	"Dual (IPv4 & IPv6)": "双重 (IPv4与IPv6)",
--- a/src/Config.py
+++ b/src/Config.py
@ -13,7 +13,7 @@ class Config(object):

    def __init__(self, argv):
        self.version = "0.7.0"
-        self.rev = 4112
+        self.rev = 4122
        self.argv = argv
        self.action = None
        self.pending_changes = {}
@ -573,7 +573,7 @@ class Config(object):
        logging.getLogger('').setLevel(logging.getLevelName(self.log_level))
        logging.getLogger('').addHandler(file_logger)

-    def initLogging(self):
+    def initLogging(self, console_logging=True, file_logging=True):
        # Create necessary files and dirs
        if not os.path.isdir(self.log_dir):
            os.mkdir(self.log_dir)
@ -589,7 +589,9 @@ class Config(object):
        logging.getLogger('').name = "-"  # Remove root prefix
        logging.getLogger("geventwebsocket.handler").setLevel(logging.WARNING)  # Don't log ws debug messages

-        self.initConsoleLogger()
-        self.initFileLogger()
+        if console_logging:
+            self.initConsoleLogger()
+        if file_logging:
+            self.initFileLogger()

 config = Config(sys.argv)
--- a/src/Db/DbCursor.py
+++ b/src/Db/DbCursor.py
@ -49,6 +49,8 @@ class DbCursor:
                    else:
                        if key.startswith("not__"):
                            query_wheres.append(key.replace("not__", "") + " != ?")
+                        elif key.endswith("__like"):
+                            query_wheres.append(key.replace("__like", "") + " LIKE ?")
                        elif key.endswith(">"):
                            query_wheres.append(key.replace(">", "") + " > ?")
                        elif key.endswith("<"):
--- a/src/Debug/DebugReloader.py
+++ b/src/Debug/DebugReloader.py
@ -1,5 +1,6 @@
 import logging
 import time
+import os

 from Config import config

@ -41,7 +42,8 @@ class DebugReloader:
        if ext not in ["py", "json"] or "Test" in path or time.time() - self.last_chaged < 1.0:
            return False
        self.last_chaged = time.time()
-        self.log.debug("File changed: %s reloading source code" % evt)
+        time_modified = os.path.getmtime(path)
+        self.log.debug("File changed: %s reloading source code (modified %.3fs ago)" % (evt, time.time() - time_modified))
        time.sleep(0.1)  # Wait for lock release
        for callback in self.callbacks:
            try:
--- a/src/Site/SiteAnnouncer.py
+++ b/src/Site/SiteAnnouncer.py
@ -1,16 +1,9 @@
 import random
 import time
 import hashlib
-import urllib.request
-import struct
-import socket
 import re
 import collections

-import bencode
-from lib.subtl.subtl import UdpTrackerClient
-import socks
-import sockshandler
 import gevent

 from Plugin import PluginManager
@ -41,8 +34,6 @@ class SiteAnnouncer(object):

    def getSupportedTrackers(self):
        trackers = self.getTrackers()
-        if config.disable_udp or config.trackers_proxy != "disable":
-            trackers = [tracker for tracker in trackers if not tracker.startswith("udp://")]

        if not self.site.connection_server.tor_manager.enabled:
            trackers = [tracker for tracker in trackers if ".onion" not in tracker]
@ -157,15 +148,7 @@ class SiteAnnouncer(object):
            self.updateWebsocket(pex="announced")

    def getTrackerHandler(self, protocol):
-        if protocol == "udp":
-            handler = self.announceTrackerUdp
-        elif protocol == "http":
-            handler = self.announceTrackerHttp
-        elif protocol == "https":
-            handler = self.announceTrackerHttps
-        else:
-            handler = None
-        return handler
+        return None

    def getAddressParts(self, tracker):
        if "://" not in tracker or not re.match("^[A-Za-z0-9:/\\.#-]+$", tracker):
@ -266,112 +249,6 @@ class SiteAnnouncer(object):
            )
        return time.time() - s

-    def announceTrackerUdp(self, tracker_address, mode="start", num_want=10):
-        s = time.time()
-        if config.disable_udp:
-            raise AnnounceError("Udp disabled by config")
-        if config.trackers_proxy != "disable":
-            raise AnnounceError("Udp trackers not available with proxies")
-
-        ip, port = tracker_address.split("/")[0].split(":")
-        tracker = UdpTrackerClient(ip, int(port))
-        if helper.getIpType(ip) in self.getOpenedServiceTypes():
-            tracker.peer_port = self.fileserver_port
-        else:
-            tracker.peer_port = 0
-        tracker.connect()
-        if not tracker.poll_once():
-            raise AnnounceError("Could not connect")
-        tracker.announce(info_hash=self.site.address_sha1, num_want=num_want, left=431102370)
-        back = tracker.poll_once()
-        if not back:
-            raise AnnounceError("No response after %.0fs" % (time.time() - s))
-        elif type(back) is dict and "response" in back:
-            peers = back["response"]["peers"]
-        else:
-            raise AnnounceError("Invalid response: %r" % back)
-
-        return peers
-
-    def httpRequest(self, url):
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
-            'Accept-Encoding': 'none',
-            'Accept-Language': 'en-US,en;q=0.8',
-            'Connection': 'keep-alive'
-        }
-
-        req = urllib.request.Request(url, headers=headers)
-
-        if config.trackers_proxy == "tor":
-            tor_manager = self.site.connection_server.tor_manager
-            handler = sockshandler.SocksiPyHandler(socks.SOCKS5, tor_manager.proxy_ip, tor_manager.proxy_port)
-            opener = urllib.request.build_opener(handler)
-            return opener.open(req, timeout=50)
-        elif config.trackers_proxy == "disable":
-            return urllib.request.urlopen(req, timeout=25)
-        else:
-            proxy_ip, proxy_port = config.trackers_proxy.split(":")
-            handler = sockshandler.SocksiPyHandler(socks.SOCKS5, proxy_ip, int(proxy_port))
-            opener = urllib.request.build_opener(handler)
-            return opener.open(req, timeout=50)
-
-    def announceTrackerHttps(self, *args, **kwargs):
-        kwargs["protocol"] = "https"
-        return self.announceTrackerHttp(*args, **kwargs)
-
-    def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protocol="http"):
-        tracker_ip, tracker_port = tracker_address.rsplit(":", 1)
-        if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes():
-            port = self.fileserver_port
-        else:
-            port = 1
-        params = {
-            'info_hash': self.site.address_sha1,
-            'peer_id': self.peer_id, 'port': port,
-            'uploaded': 0, 'downloaded': 0, 'left': 431102370, 'compact': 1, 'numwant': num_want,
-            'event': 'started'
-        }
-
-        url = protocol + "://" + tracker_address + "?" + urllib.parse.urlencode(params)
-
-        s = time.time()
-        response = None
-        # Load url
-        if config.tor == "always" or config.trackers_proxy != "disable":
-            timeout = 60
-        else:
-            timeout = 30
-
-        with gevent.Timeout(timeout, False):  # Make sure of timeout
-            req = self.httpRequest(url)
-            response = req.read()
-            req.close()
-            req = None
-
-        if not response:
-            raise AnnounceError("No response after %.0fs" % (time.time() - s))
-
-        # Decode peers
-        try:
-            peer_data = bencode.decode(response)["peers"]
-            if type(peer_data) is not bytes:
-                peer_data = peer_data.encode()
-            response = None
-            peer_count = int(len(peer_data) / 6)
-            peers = []
-            for peer_offset in range(peer_count):
-                off = 6 * peer_offset
-                peer = peer_data[off:off + 6]
-                addr, port = struct.unpack('!LH', peer)
-                peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port})
-        except Exception as err:
-            raise AnnounceError("Invalid response: %r (%s)" % (response, Debug.formatException(err)))
-
-        return peers
-
    @util.Noparallel(blocking=False)
    def announcePex(self, query_num=2, need_num=5):
        peers = self.site.getConnectedPeers()
--- a/src/Test/TestDb.py
+++ b/src/Test/TestDb.py
@ -75,6 +75,11 @@ class TestDb:
            {"not__title": ["Test #%s" % i for i in range(50, 3000)]}
        ).fetchone()["num"] == 50

+        assert db.execute(
+            "SELECT COUNT(*) AS num FROM test WHERE ?",
+            {"title__like": "%20%"}
+        ).fetchone()["num"] == 1
+
        # Test named parameter escaping
        assert db.execute(
            "SELECT COUNT(*) AS num FROM test WHERE test_id = :test_id AND title LIKE :titlelike",
--- a/src/Test/TestFileRequest.py
+++ b/src/Test/TestFileRequest.py
@ -48,6 +48,12 @@ class TestFileRequest:
        response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0, "file_size": 1234})
        assert "File size does not match" in response["error"]

+        # Invalid path
+        for path in ["../users.json", "./../users.json", "data/../content.json", ".../users.json"]:
+            for sep in ["/", "\\"]:
+                response = connection.request("getFile", {"site": site.address, "inner_path": path.replace("/", sep), "location": 0})
+                assert response["error"] == 'File read exception'
+
        connection.close()
        client.stop()

--- a/src/Test/conftest.py
+++ b/src/Test/conftest.py
@ -68,7 +68,7 @@ config.verbose = True  # Use test data for unittests
 config.tor = "disable"  # Don't start Tor client
 config.trackers = []
 config.data_dir = TEST_DATA_PATH  # Use test data for unittests
-config.initLogging()
+config.initLogging(console_logging=False)

 # Set custom formatter with realative time format (via: https://stackoverflow.com/questions/31521859/python-logging-module-time-since-last-log)
 class TimeFilter(logging.Filter):
--- a/src/Ui/media/ZeroSiteTheme.coffee
+++ b/src/Ui/media/ZeroSiteTheme.coffee
@ -9,10 +9,10 @@ changeColorScheme = (theme) ->
    zeroframe.cmd "userGetGlobalSettings", [], (user_settings) ->
        if user_settings.theme != theme
            user_settings.theme = theme
-            zeroframe.cmd "userSetGlobalSettings", [user_settings]
-
-            location.reload()
-
+            zeroframe.cmd "userSetGlobalSettings", [user_settings], (status) ->
+                if status == "ok"
+                    location.reload()
+                return
        return
    return

@ -21,7 +21,12 @@ displayNotification = ({matches, media}) ->
    if !matches
        return

-    zeroframe.cmd "wrapperNotification", ["info", "Your system's theme has been changed.<br>Please reload site to use it."]
+    zeroframe.cmd "siteInfo", [], (site_info) ->
+        if "ADMIN" in site_info.settings.permissions
+            zeroframe.cmd "wrapperNotification", ["info", "Your system's theme has been changed.<br>Please reload site to use it."]
+        else
+            zeroframe.cmd "wrapperNotification", ["info", "Your system's theme has been changed.<br>Please open ZeroHello to use it."]
+        return
    return


--- a/src/Ui/media/all.js
+++ b/src/Ui/media/all.js
@ -1981,7 +1981,6 @@ $.extend( $.easing,

 }).call(this);

-
 /* ---- src/Ui/media/WrapperZeroFrame.coffee ---- */


@ -2037,7 +2036,8 @@ $.extend( $.easing,


 (function() {
-  var DARK, LIGHT, changeColorScheme, detectColorScheme, displayNotification, mqDark, mqLight;
+  var DARK, LIGHT, changeColorScheme, detectColorScheme, displayNotification, mqDark, mqLight,
+    indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; };

  DARK = "(prefers-color-scheme: dark)";

@ -2051,8 +2051,11 @@ $.extend( $.easing,
    zeroframe.cmd("userGetGlobalSettings", [], function(user_settings) {
      if (user_settings.theme !== theme) {
        user_settings.theme = theme;
-        zeroframe.cmd("userSetGlobalSettings", [user_settings]);
-        location.reload();
+        zeroframe.cmd("userSetGlobalSettings", [user_settings], function(status) {
+          if (status === "ok") {
+            location.reload();
+          }
+        });
      }
    });
  };
@ -2063,7 +2066,13 @@ $.extend( $.easing,
    if (!matches) {
      return;
    }
-    zeroframe.cmd("wrapperNotification", ["info", "Your system's theme has been changed.<br>Please reload site to use it."]);
+    zeroframe.cmd("siteInfo", [], function(site_info) {
+      if (indexOf.call(site_info.settings.permissions, "ADMIN") >= 0) {
+        zeroframe.cmd("wrapperNotification", ["info", "Your system's theme has been changed.<br>Please reload site to use it."]);
+      } else {
+        zeroframe.cmd("wrapperNotification", ["info", "Your system's theme has been changed.<br>Please open ZeroHello to use it."]);
+      }
+    });
  };

  detectColorScheme = function() {