diff --git a/.travis.yml b/.travis.yml index e244b8fc..d8821001 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ before_install: # - docker build -t zeronet . # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet install: - - if [[ $TRAVIS_PYTHON_VERSION == 3.8-dev ]]; then pip install --upgrade setuptools cffi 'cython>=0.28' git+git://github.com/gevent/gevent.git#egg=gevent; fi - pip install --upgrade -r requirements.txt - pip list before_script: diff --git a/README.md b/README.md index 708116e3..4b981863 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -__Warning: Development test version, do not use on live data__ - # ZeroNet [![Build Status](https://travis-ci.org/HelloZeroNet/ZeroNet.svg?branch=master)](https://travis-ci.org/HelloZeroNet/ZeroNet) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://zeronet.io/docs/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://zeronet.io/docs/help_zeronet/donate/) Decentralized websites using Bitcoin crypto and the BitTorrent network - https://zeronet.io @@ -63,19 +61,18 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/ ## How to join -### Install from package for your distribution +### Windows -* Arch Linux: [zeronet](https://aur.archlinux.org/zeronet.git), [zeronet-git](https://aur.archlinux.org/zeronet-git.git) -* Gentoo: [emerge repository](https://github.com/leycec/raiagent) -* FreeBSD: zeronet -* Whonix: [instructions](https://www.whonix.org/wiki/ZeroNet) + - Download [ZeroNet-py3-win64.zip](https://github.com/HelloZeroNet/ZeroNet-win/archive/dist-win64/ZeroNet-py3-win64.zip) (18MB) + - Unpack anywhere + - Run `ZeroNet.exe` -### Install from source +### Other platforms: Install from source Fetch and extract the source: - wget https://github.com/HelloZeroNet/ZeroNet/archive/py3.tar.gz - tar xvpfz py3.tar.gz + wget https://github.com/HelloZeroNet/ZeroNet/archive/py3/ZeroNet-py3.tar.gz + tar xvpfz ZeroNet-py3.tar.gz cd ZeroNet-py3 Install Python module dependencies either: diff --git a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py new file mode 100644 index 00000000..ae674c00 --- /dev/null +++ b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py @@ -0,0 +1,150 @@ +import time +import urllib.request +import struct +import socket + +import bencode +from lib.subtl.subtl import UdpTrackerClient +import socks +import sockshandler +import gevent + +from Plugin import PluginManager +from Config import config +from Debug import Debug +from util import helper + + +# We can only import plugin host clases after the plugins are loaded +@PluginManager.afterLoad +def importHostClasses(): + global Peer, AnnounceError + from Peer import Peer + from Site.SiteAnnouncer import AnnounceError + + +@PluginManager.registerTo("SiteAnnouncer") +class SiteAnnouncerPlugin(object): + def getSupportedTrackers(self): + trackers = super(SiteAnnouncerPlugin, self).getSupportedTrackers() + if config.disable_udp or config.trackers_proxy != "disable": + trackers = [tracker for tracker in trackers if not tracker.startswith("udp://")] + + return trackers + + def getTrackerHandler(self, protocol): + if protocol == "udp": + handler = self.announceTrackerUdp + elif protocol == "http": + handler = self.announceTrackerHttp + elif protocol == "https": + handler = self.announceTrackerHttps + else: + handler = super(SiteAnnouncerPlugin, self).getTrackerHandler(protocol) + return handler + + def announceTrackerUdp(self, tracker_address, mode="start", num_want=10): + s = time.time() + if config.disable_udp: + raise AnnounceError("Udp disabled by config") + if config.trackers_proxy != "disable": + raise AnnounceError("Udp trackers not available with proxies") + + ip, port = tracker_address.split("/")[0].split(":") + tracker = UdpTrackerClient(ip, int(port)) + if helper.getIpType(ip) in self.getOpenedServiceTypes(): + tracker.peer_port = self.fileserver_port + else: + tracker.peer_port = 0 + tracker.connect() + if not tracker.poll_once(): + raise AnnounceError("Could not connect") + tracker.announce(info_hash=self.site.address_sha1, num_want=num_want, left=431102370) + back = tracker.poll_once() + if not back: + raise AnnounceError("No response after %.0fs" % (time.time() - s)) + elif type(back) is dict and "response" in back: + peers = back["response"]["peers"] + else: + raise AnnounceError("Invalid response: %r" % back) + + return peers + + def httpRequest(self, url): + headers = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', + 'Accept-Encoding': 'none', + 'Accept-Language': 'en-US,en;q=0.8', + 'Connection': 'keep-alive' + } + + req = urllib.request.Request(url, headers=headers) + + if config.trackers_proxy == "tor": + tor_manager = self.site.connection_server.tor_manager + handler = sockshandler.SocksiPyHandler(socks.SOCKS5, tor_manager.proxy_ip, tor_manager.proxy_port) + opener = urllib.request.build_opener(handler) + return opener.open(req, timeout=50) + elif config.trackers_proxy == "disable": + return urllib.request.urlopen(req, timeout=25) + else: + proxy_ip, proxy_port = config.trackers_proxy.split(":") + handler = sockshandler.SocksiPyHandler(socks.SOCKS5, proxy_ip, int(proxy_port)) + opener = urllib.request.build_opener(handler) + return opener.open(req, timeout=50) + + def announceTrackerHttps(self, *args, **kwargs): + kwargs["protocol"] = "https" + return self.announceTrackerHttp(*args, **kwargs) + + def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protocol="http"): + tracker_ip, tracker_port = tracker_address.rsplit(":", 1) + if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes(): + port = self.fileserver_port + else: + port = 1 + params = { + 'info_hash': self.site.address_sha1, + 'peer_id': self.peer_id, 'port': port, + 'uploaded': 0, 'downloaded': 0, 'left': 431102370, 'compact': 1, 'numwant': num_want, + 'event': 'started' + } + + url = protocol + "://" + tracker_address + "?" + urllib.parse.urlencode(params) + + s = time.time() + response = None + # Load url + if config.tor == "always" or config.trackers_proxy != "disable": + timeout = 60 + else: + timeout = 30 + + with gevent.Timeout(timeout, False): # Make sure of timeout + req = self.httpRequest(url) + response = req.read() + req.close() + req = None + + if not response: + raise AnnounceError("No response after %.0fs" % (time.time() - s)) + + # Decode peers + try: + peer_data = bencode.decode(response)["peers"] + if type(peer_data) is not bytes: + peer_data = peer_data.encode() + response = None + peer_count = int(len(peer_data) / 6) + peers = [] + for peer_offset in range(peer_count): + off = 6 * peer_offset + peer = peer_data[off:off + 6] + addr, port = struct.unpack('!LH', peer) + peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port}) + except Exception as err: + raise AnnounceError("Invalid response: %r (%s)" % (response, Debug.formatException(err))) + + return peers diff --git a/plugins/AnnounceBitTorrent/__init__.py b/plugins/AnnounceBitTorrent/__init__.py new file mode 100644 index 00000000..c7422855 --- /dev/null +++ b/plugins/AnnounceBitTorrent/__init__.py @@ -0,0 +1 @@ +from . import AnnounceBitTorrentPlugin \ No newline at end of file diff --git a/plugins/OptionalManager/UiWebsocketPlugin.py b/plugins/OptionalManager/UiWebsocketPlugin.py index 52b0f749..626dc2e1 100644 --- a/plugins/OptionalManager/UiWebsocketPlugin.py +++ b/plugins/OptionalManager/UiWebsocketPlugin.py @@ -105,7 +105,7 @@ class UiWebsocketPlugin(object): # Optional file functions - def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC", limit=10, filter="downloaded"): + def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC", limit=10, filter="downloaded", filter_inner_path=None): if not address: address = self.site.address @@ -139,6 +139,8 @@ class UiWebsocketPlugin(object): wheres_raw.append("(is_downloaded = 1 OR is_pinned = 1)") if "pinned" in filter: wheres["is_pinned"] = 1 + if filter_inner_path: + wheres["inner_path__like"] = filter_inner_path if address == "all": join = "LEFT JOIN site USING (site_id)" diff --git a/plugins/UiConfig/languages/zh.json b/plugins/UiConfig/languages/zh.json index 6e58a59c..6d6e9f80 100644 --- a/plugins/UiConfig/languages/zh.json +++ b/plugins/UiConfig/languages/zh.json @@ -4,6 +4,8 @@ "Open web browser on ZeroNet startup": "ZeroNet启动时,打开浏览器", "Network": "网络", + "Offline mode": "离线模式", + "Disable network communication.": "关闭网络通信.", "File server network": "文件服务器网络", "Accept incoming peers using IPv4 or IPv6 address. (default: dual)": "使用IPv4或IPv6地址接受传入的节点请求. (默认:双重)", "Dual (IPv4 & IPv6)": "双重 (IPv4与IPv6)", diff --git a/src/Config.py b/src/Config.py index 1f9dcaee..13a8ec44 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.7.0" - self.rev = 4112 + self.rev = 4122 self.argv = argv self.action = None self.pending_changes = {} @@ -573,7 +573,7 @@ class Config(object): logging.getLogger('').setLevel(logging.getLevelName(self.log_level)) logging.getLogger('').addHandler(file_logger) - def initLogging(self): + def initLogging(self, console_logging=True, file_logging=True): # Create necessary files and dirs if not os.path.isdir(self.log_dir): os.mkdir(self.log_dir) @@ -589,7 +589,9 @@ class Config(object): logging.getLogger('').name = "-" # Remove root prefix logging.getLogger("geventwebsocket.handler").setLevel(logging.WARNING) # Don't log ws debug messages - self.initConsoleLogger() - self.initFileLogger() + if console_logging: + self.initConsoleLogger() + if file_logging: + self.initFileLogger() config = Config(sys.argv) diff --git a/src/Db/DbCursor.py b/src/Db/DbCursor.py index 274782ec..2c41c2c1 100644 --- a/src/Db/DbCursor.py +++ b/src/Db/DbCursor.py @@ -49,6 +49,8 @@ class DbCursor: else: if key.startswith("not__"): query_wheres.append(key.replace("not__", "") + " != ?") + elif key.endswith("__like"): + query_wheres.append(key.replace("__like", "") + " LIKE ?") elif key.endswith(">"): query_wheres.append(key.replace(">", "") + " > ?") elif key.endswith("<"): diff --git a/src/Debug/DebugReloader.py b/src/Debug/DebugReloader.py index ffb18ce3..e6cdec06 100644 --- a/src/Debug/DebugReloader.py +++ b/src/Debug/DebugReloader.py @@ -1,5 +1,6 @@ import logging import time +import os from Config import config @@ -41,7 +42,8 @@ class DebugReloader: if ext not in ["py", "json"] or "Test" in path or time.time() - self.last_chaged < 1.0: return False self.last_chaged = time.time() - self.log.debug("File changed: %s reloading source code" % evt) + time_modified = os.path.getmtime(path) + self.log.debug("File changed: %s reloading source code (modified %.3fs ago)" % (evt, time.time() - time_modified)) time.sleep(0.1) # Wait for lock release for callback in self.callbacks: try: diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index f066a033..2e2a353f 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -1,16 +1,9 @@ import random import time import hashlib -import urllib.request -import struct -import socket import re import collections -import bencode -from lib.subtl.subtl import UdpTrackerClient -import socks -import sockshandler import gevent from Plugin import PluginManager @@ -41,8 +34,6 @@ class SiteAnnouncer(object): def getSupportedTrackers(self): trackers = self.getTrackers() - if config.disable_udp or config.trackers_proxy != "disable": - trackers = [tracker for tracker in trackers if not tracker.startswith("udp://")] if not self.site.connection_server.tor_manager.enabled: trackers = [tracker for tracker in trackers if ".onion" not in tracker] @@ -157,15 +148,7 @@ class SiteAnnouncer(object): self.updateWebsocket(pex="announced") def getTrackerHandler(self, protocol): - if protocol == "udp": - handler = self.announceTrackerUdp - elif protocol == "http": - handler = self.announceTrackerHttp - elif protocol == "https": - handler = self.announceTrackerHttps - else: - handler = None - return handler + return None def getAddressParts(self, tracker): if "://" not in tracker or not re.match("^[A-Za-z0-9:/\\.#-]+$", tracker): @@ -266,112 +249,6 @@ class SiteAnnouncer(object): ) return time.time() - s - def announceTrackerUdp(self, tracker_address, mode="start", num_want=10): - s = time.time() - if config.disable_udp: - raise AnnounceError("Udp disabled by config") - if config.trackers_proxy != "disable": - raise AnnounceError("Udp trackers not available with proxies") - - ip, port = tracker_address.split("/")[0].split(":") - tracker = UdpTrackerClient(ip, int(port)) - if helper.getIpType(ip) in self.getOpenedServiceTypes(): - tracker.peer_port = self.fileserver_port - else: - tracker.peer_port = 0 - tracker.connect() - if not tracker.poll_once(): - raise AnnounceError("Could not connect") - tracker.announce(info_hash=self.site.address_sha1, num_want=num_want, left=431102370) - back = tracker.poll_once() - if not back: - raise AnnounceError("No response after %.0fs" % (time.time() - s)) - elif type(back) is dict and "response" in back: - peers = back["response"]["peers"] - else: - raise AnnounceError("Invalid response: %r" % back) - - return peers - - def httpRequest(self, url): - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', - 'Accept-Encoding': 'none', - 'Accept-Language': 'en-US,en;q=0.8', - 'Connection': 'keep-alive' - } - - req = urllib.request.Request(url, headers=headers) - - if config.trackers_proxy == "tor": - tor_manager = self.site.connection_server.tor_manager - handler = sockshandler.SocksiPyHandler(socks.SOCKS5, tor_manager.proxy_ip, tor_manager.proxy_port) - opener = urllib.request.build_opener(handler) - return opener.open(req, timeout=50) - elif config.trackers_proxy == "disable": - return urllib.request.urlopen(req, timeout=25) - else: - proxy_ip, proxy_port = config.trackers_proxy.split(":") - handler = sockshandler.SocksiPyHandler(socks.SOCKS5, proxy_ip, int(proxy_port)) - opener = urllib.request.build_opener(handler) - return opener.open(req, timeout=50) - - def announceTrackerHttps(self, *args, **kwargs): - kwargs["protocol"] = "https" - return self.announceTrackerHttp(*args, **kwargs) - - def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protocol="http"): - tracker_ip, tracker_port = tracker_address.rsplit(":", 1) - if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes(): - port = self.fileserver_port - else: - port = 1 - params = { - 'info_hash': self.site.address_sha1, - 'peer_id': self.peer_id, 'port': port, - 'uploaded': 0, 'downloaded': 0, 'left': 431102370, 'compact': 1, 'numwant': num_want, - 'event': 'started' - } - - url = protocol + "://" + tracker_address + "?" + urllib.parse.urlencode(params) - - s = time.time() - response = None - # Load url - if config.tor == "always" or config.trackers_proxy != "disable": - timeout = 60 - else: - timeout = 30 - - with gevent.Timeout(timeout, False): # Make sure of timeout - req = self.httpRequest(url) - response = req.read() - req.close() - req = None - - if not response: - raise AnnounceError("No response after %.0fs" % (time.time() - s)) - - # Decode peers - try: - peer_data = bencode.decode(response)["peers"] - if type(peer_data) is not bytes: - peer_data = peer_data.encode() - response = None - peer_count = int(len(peer_data) / 6) - peers = [] - for peer_offset in range(peer_count): - off = 6 * peer_offset - peer = peer_data[off:off + 6] - addr, port = struct.unpack('!LH', peer) - peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port}) - except Exception as err: - raise AnnounceError("Invalid response: %r (%s)" % (response, Debug.formatException(err))) - - return peers - @util.Noparallel(blocking=False) def announcePex(self, query_num=2, need_num=5): peers = self.site.getConnectedPeers() diff --git a/src/Test/TestDb.py b/src/Test/TestDb.py index dda2ca20..67f383a3 100644 --- a/src/Test/TestDb.py +++ b/src/Test/TestDb.py @@ -75,6 +75,11 @@ class TestDb: {"not__title": ["Test #%s" % i for i in range(50, 3000)]} ).fetchone()["num"] == 50 + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"title__like": "%20%"} + ).fetchone()["num"] == 1 + # Test named parameter escaping assert db.execute( "SELECT COUNT(*) AS num FROM test WHERE test_id = :test_id AND title LIKE :titlelike", diff --git a/src/Test/TestFileRequest.py b/src/Test/TestFileRequest.py index ef28ee96..6a8d634a 100644 --- a/src/Test/TestFileRequest.py +++ b/src/Test/TestFileRequest.py @@ -48,6 +48,12 @@ class TestFileRequest: response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0, "file_size": 1234}) assert "File size does not match" in response["error"] + # Invalid path + for path in ["../users.json", "./../users.json", "data/../content.json", ".../users.json"]: + for sep in ["/", "\\"]: + response = connection.request("getFile", {"site": site.address, "inner_path": path.replace("/", sep), "location": 0}) + assert response["error"] == 'File read exception' + connection.close() client.stop() diff --git a/src/Test/conftest.py b/src/Test/conftest.py index b4c96d0e..1f068d69 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -68,7 +68,7 @@ config.verbose = True # Use test data for unittests config.tor = "disable" # Don't start Tor client config.trackers = [] config.data_dir = TEST_DATA_PATH # Use test data for unittests -config.initLogging() +config.initLogging(console_logging=False) # Set custom formatter with realative time format (via: https://stackoverflow.com/questions/31521859/python-logging-module-time-since-last-log) class TimeFilter(logging.Filter): diff --git a/src/Ui/media/ZeroSiteTheme.coffee b/src/Ui/media/ZeroSiteTheme.coffee index 47144051..79adb671 100644 --- a/src/Ui/media/ZeroSiteTheme.coffee +++ b/src/Ui/media/ZeroSiteTheme.coffee @@ -9,10 +9,10 @@ changeColorScheme = (theme) -> zeroframe.cmd "userGetGlobalSettings", [], (user_settings) -> if user_settings.theme != theme user_settings.theme = theme - zeroframe.cmd "userSetGlobalSettings", [user_settings] - - location.reload() - + zeroframe.cmd "userSetGlobalSettings", [user_settings], (status) -> + if status == "ok" + location.reload() + return return return @@ -21,7 +21,12 @@ displayNotification = ({matches, media}) -> if !matches return - zeroframe.cmd "wrapperNotification", ["info", "Your system's theme has been changed.
Please reload site to use it."] + zeroframe.cmd "siteInfo", [], (site_info) -> + if "ADMIN" in site_info.settings.permissions + zeroframe.cmd "wrapperNotification", ["info", "Your system's theme has been changed.
Please reload site to use it."] + else + zeroframe.cmd "wrapperNotification", ["info", "Your system's theme has been changed.
Please open ZeroHello to use it."] + return return diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index fc7d6fda..86c61b51 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -1981,7 +1981,6 @@ $.extend( $.easing, }).call(this); - /* ---- src/Ui/media/WrapperZeroFrame.coffee ---- */ @@ -2037,7 +2036,8 @@ $.extend( $.easing, (function() { - var DARK, LIGHT, changeColorScheme, detectColorScheme, displayNotification, mqDark, mqLight; + var DARK, LIGHT, changeColorScheme, detectColorScheme, displayNotification, mqDark, mqLight, + indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; }; DARK = "(prefers-color-scheme: dark)"; @@ -2051,8 +2051,11 @@ $.extend( $.easing, zeroframe.cmd("userGetGlobalSettings", [], function(user_settings) { if (user_settings.theme !== theme) { user_settings.theme = theme; - zeroframe.cmd("userSetGlobalSettings", [user_settings]); - location.reload(); + zeroframe.cmd("userSetGlobalSettings", [user_settings], function(status) { + if (status === "ok") { + location.reload(); + } + }); } }); }; @@ -2063,7 +2066,13 @@ $.extend( $.easing, if (!matches) { return; } - zeroframe.cmd("wrapperNotification", ["info", "Your system's theme has been changed.
Please reload site to use it."]); + zeroframe.cmd("siteInfo", [], function(site_info) { + if (indexOf.call(site_info.settings.permissions, "ADMIN") >= 0) { + zeroframe.cmd("wrapperNotification", ["info", "Your system's theme has been changed.
Please reload site to use it."]); + } else { + zeroframe.cmd("wrapperNotification", ["info", "Your system's theme has been changed.
Please open ZeroHello to use it."]); + } + }); }; detectColorScheme = function() { @@ -2082,4 +2091,4 @@ $.extend( $.easing, } }); -}).call(this); \ No newline at end of file +}).call(this);