From 1c5b5e36211908fb47582ec922f350f9ed669b76 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sun, 20 Jan 2019 03:13:54 +0100 Subject: [PATCH] Move port checking to separate file, add ipv6 port check --- src/File/FileServer.py | 199 +++++++----------------------------- src/Peer/PeerPortchecker.py | 175 +++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 160 deletions(-) create mode 100644 src/Peer/PeerPortchecker.py diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 3e207c46..c4cfe0ca 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -1,20 +1,17 @@ import logging -import urllib2 -import re import time import random -import socket import gevent import gevent.pool import util +from util import helper from Config import config from FileRequest import FileRequest +from Peer import PeerPortchecker from Site import SiteManager -from Debug import Debug from Connection import ConnectionServer -from util import UpnpPunch from Plugin import PluginManager @@ -23,6 +20,7 @@ class FileServer(ConnectionServer): def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port): self.site_manager = SiteManager.site_manager + self.portchecker = PeerPortchecker.PeerPortchecker(self) self.log = logging.getLogger("FileServer") ip = ip.replace("*", "0.0.0.0") @@ -44,8 +42,7 @@ class FileServer(ConnectionServer): self.port_opened = True SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist else: - self.port_opened = None # Is file server opened on router - self.upnp_port_opened = False + self.port_opened = None # Check it later self.sites = {} self.last_request = time.time() self.files_parsing = {} @@ -59,7 +56,7 @@ class FileServer(ConnectionServer): if port in tried: continue tried.append(port) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock = helper.createSocket(ip) try: sock.bind((ip, port)) success = True @@ -68,6 +65,7 @@ class FileServer(ConnectionServer): success = False sock.close() if success: + self.log.info("Found unused random port: %s" % port) return port else: time.sleep(0.1) @@ -99,157 +97,31 @@ class FileServer(ConnectionServer): import imp FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest - # Try to open the port using upnp - def openport(self, port=None, check=True): - if not port: - port = self.port - if self.port_opened: - return True # Port already opened - if check: # Check first if its already opened - time.sleep(1) # Wait for port open - if self.testOpenport(port, use_alternative=False)["result"] is True: - return True # Port already opened - - if config.tor == "always": # Port opening won't work in Tor mode - return False - - self.log.info("Trying to open port using UpnpPunch...") - try: - UpnpPunch.ask_to_open_port(self.port, 'ZeroNet', retries=3, protos=["TCP"]) - except Exception as err: - self.log.warning("UpnpPunch run error: %s" % Debug.formatException(err)) - return False - - if self.testOpenport(port)["result"] is True: - self.upnp_port_opened = True - return True - else: - self.log.info("Upnp mapping failed :( Please forward port %s on your router to your ipaddress" % port) - return False - - # Test if the port is open - def testOpenport(self, port=None, use_alternative=True): - if not port: - port = self.port - back = self.testOpenportPortchecker(port) - if (back["result"] is not True and use_alternative) or back["result"] is None: # If no success try alternative checker - back = self.testOpenportCanyouseeme(port) - - if self.ui_server: - self.ui_server.updateWebsocket() - - return back - - def testOpenportP2P(self, port=None): - self.log.info("Checking port %s using P2P..." % port) - site = self.site_manager.get(config.homepage) - peers = [] - res = None - if not site: # First run, has no any peers - return self.testOpenportPortchecker(port) # Fallback to centralized service - peers = [peer for peer in site.getRecentPeers(10) if not peer.ip.endswith(".onion")] - if len(peers) < 3: # Not enough peers - return self.testOpenportPortchecker(port) # Fallback to centralized service - for retry in range(0, 3): # Try 3 peers - random_peer = random.choice(peers) - with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception - if not random_peer.connection: - random_peer.connect() - if random_peer.connection and random_peer.connection.handshake.get("rev") >= 2186: - res = random_peer.request("checkport", {"port": port}) - if res is not None: - break # All fine, exit from for loop - - if res is None: # Nobody answered - return self.testOpenportPortchecker(port) # Fallback to centralized service - if res["status"] == "closed": - if config.tor != "always": - self.log.info("[BAD :(] %s says that your port %s is closed" % (random_peer.ip, port)) - if port == self.port: - self.port_opened = False # Self port, update port_opened status - config.ip_external = res["ip_external"] - SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist - return {"result": False} - else: - self.log.info("[OK :)] %s says that your port %s is open" % (random_peer.ip, port)) - if port == self.port: # Self port, update port_opened status - self.port_opened = True - config.ip_external = res["ip_external"] - SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist - return {"result": True} - - def testOpenportPortchecker(self, port=None): - self.log.info("Checking port %s using portchecker.co..." % port) - try: - data = urllib2.urlopen("https://portchecker.co/check", "port=%s" % port, timeout=20.0).read() - message = re.match('.*
(.*?)
', data, re.DOTALL).group(1) - message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags - except Exception, err: - return {"result": None, "message": Debug.formatException(err)} - - if "open" not in message: - if config.tor != "always": - self.log.info("[BAD :(] Port closed: %s" % message) - if port == self.port: - self.port_opened = False # Self port, update port_opened status - match = re.match(".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL) # Try find my external ip in message - if match: # Found my ip in message - config.ip_external = match.group(1) - SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist - else: - config.ip_external = False - return {"result": False, "message": message} - else: - self.log.info("[OK :)] Port open: %s" % message) - if port == self.port: # Self port, update port_opened status - self.port_opened = True - match = re.match(".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL) # Try find my external ip in message - if match: # Found my ip in message - config.ip_external = match.group(1) - SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist - else: - config.ip_external = False - return {"result": True, "message": message} - - def testOpenportCanyouseeme(self, port=None): - self.log.info("Checking port %s using canyouseeme.org..." % port) - try: - data = urllib2.urlopen("http://www.canyouseeme.org/", "port=%s" % port, timeout=20.0).read() - message = re.match('.*

(.*?)

', data, re.DOTALL).group(1) - message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ")) # Strip http tags - except Exception, err: - return {"result": None, "message": Debug.formatException(err)} - - if "Success" not in message: - if config.tor != "always": - self.log.info("[BAD :(] Port closed: %s" % message) - if port == self.port: - self.port_opened = False # Self port, update port_opened status - match = re.match(".*?([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)", message) # Try find my external ip in message - if match: # Found my ip in message - config.ip_external = match.group(1) - SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist - else: - config.ip_external = False - return {"result": False, "message": message} - else: - self.log.info("[OK :)] Port open: %s" % message) - if port == self.port: # Self port, update port_opened status - self.port_opened = True - match = re.match(".*?([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)", message) # Try find my external ip in message - if match: # Found my ip in message - config.ip_external = match.group(1) - SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist - else: - config.ip_external = False - return {"result": True, "message": message} - # Set external ip without testing def setIpExternal(self, ip_external): logging.info("Setting external ip without testing: %s..." % ip_external) config.ip_external = ip_external self.port_opened = True + def portCheck(self): + res = self.portchecker.portCheck(self.port, helper.getIpType(self.ip)) + if not res["opened"]: + if self.portchecker.portOpen(self.port): + res = self.portchecker.portCheck(self.port, helper.getIpType(self.ip)) + + if res["ip"]: + config.ip_external = res["ip"] + SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist + else: + config.ip_external = False + + if res["opened"]: + self.log.info("Server port on %s:%s: Open" % (self.ip, self.port)) + return True + else: + self.log.info("Server port on %s:%s: Closed" % (self.ip, self.port)) + return False + # Check site file integrity def checkSite(self, site, check_files=False): if site.settings["serving"]: @@ -272,13 +144,18 @@ class FileServer(ConnectionServer): if force_port_check: self.port_opened = None - self.openport() + + self.port_opened = self.portCheck() + if self.port_opened is False: self.tor_manager.startOnions() if not sites_checking: check_pool = gevent.pool.Pool(5) - for site in sorted(self.sites.values(), key=lambda site: site.settings.get("modified", 0), reverse=True): # Check sites integrity + # Check sites integrity + for site in sorted(self.sites.values(), key=lambda site: site.settings.get("modified", 0), reverse=True): + if not site.settings["serving"]: + continue check_thread = check_pool.spawn(self.checkSite, site, check_files) # Check in new thread time.sleep(2) if site.settings.get("modified", 0) < time.time() - 60 * 60 * 24: # Not so active site, wait some sec to finish @@ -292,7 +169,10 @@ class FileServer(ConnectionServer): peers_protected = set([]) while 1: # Sites health care every 20 min - self.log.debug("Running site cleanup, connections: %s, internet: %s, protected peers: %s" % (len(self.connections), self.has_internet, peers_protected)) + self.log.debug( + "Running site cleanup, connections: %s, internet: %s, protected peers: %s" % + (len(self.connections), self.has_internet, len(peers_protected)) + ) for address, site in self.sites.items(): if not site.settings["serving"]: @@ -378,7 +258,6 @@ class FileServer(ConnectionServer): from Debug import DebugReloader DebugReloader(self.reload) - if check_sites: # Open port, Update sites, Check files integrity gevent.spawn(self.checkSites) @@ -391,12 +270,12 @@ class FileServer(ConnectionServer): self.log.debug("Stopped.") def stop(self): - if self.running and self.upnp_port_opened: + if self.running and self.portchecker.upnp_port_opened: self.log.debug('Closing port %d' % self.port) try: - UpnpPunch.ask_to_close_port(self.port, protos=["TCP"]) + self.portchecker.portClose(self.port) self.log.info('Closed port via upnp.') - except (UpnpPunch.UpnpError, UpnpPunch.IGDError), err: + except Exception as err: self.log.info("Failed at attempt to use upnp to close port: %s" % err) return ConnectionServer.stop(self) diff --git a/src/Peer/PeerPortchecker.py b/src/Peer/PeerPortchecker.py new file mode 100644 index 00000000..2dcee7d3 --- /dev/null +++ b/src/Peer/PeerPortchecker.py @@ -0,0 +1,175 @@ +import logging +import urllib +import urllib2 +import re +import time + +from Debug import Debug +from util import UpnpPunch + + +class PeerPortchecker(object): + def __init__(self, file_server): + self.log = logging.getLogger("PeerPortchecker") + self.upnp_port_opened = False + self.file_server = file_server + + def requestUrl(self, url, post_data=None): + if type(post_data) is dict: + post_data = urllib.urlencode(post_data) + req = urllib2.Request(url, post_data) + req.add_header('Referer', url) + return urllib2.urlopen(req, timeout=20.0) + + def portOpen(self, port): + self.log.info("Trying to open port using UpnpPunch...") + + try: + UpnpPunch.ask_to_open_port(port, 'ZeroNet', retries=3, protos=["TCP"]) + self.upnp_port_opened = True + except Exception as err: + self.log.warning("UpnpPunch run error: %s" % Debug.formatException(err)) + return False + + if self.portCheck(port)["opened"]: + return True + else: + self.log.info("Upnp mapping failed, please forward port %s on your router to your ipaddress" % port) + return False + + def portClose(self, port): + return UpnpPunch.ask_to_close_port(port, protos=["TCP"]) + + def portCheck(self, port, ip_type="ipv4"): + if ip_type == "ipv6": + checker_functions = ["checkMyaddr", "checkIpv6scanner"] + else: + checker_functions = ["checkPortchecker", "checkCanyouseeme"] + + for func_name in checker_functions: + func = getattr(self, func_name) + s = time.time() + try: + res = func(port) + if res: + self.log.info( + "Checking port %s (%s) using %s result: %s in %.3fs" % + (port, ip_type, func_name, res, time.time() - s) + ) + time.sleep(0.1) + if res["opened"] and not self.file_server.had_external_incoming: + res["opened"] = False + self.log.warning("Port %s:%s, but no incoming connection" % (res["ip"], port)) + break + except Exception as err: + self.log.warning( + "%s check error: %s in %.3fs" % + (func_name, Debug.formatException(err), time.time() - s) + ) + res = {"ip": None, "opened": True} + + return res + + def checkCanyouseeme(self, port): + data = urllib2.urlopen("http://www.canyouseeme.org/", "port=%s" % port, timeout=20.0).read() + message = re.match('.*

(.*?)

', data, re.DOTALL).group(1) + message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ")) # Strip http tags + + match = re.match(".*service on (.*?) on", message) + if match: + ip = match.group(1) + else: + raise Exception("Invalid response: %s" % message) + + if "Success" in message: + return {"ip": ip, "opened": True} + elif "Error" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkPortchecker(self, port): + data = urllib2.urlopen("https://portchecker.co/check", "port=%s" % port, timeout=20.0).read() + message = re.match('.*
(.*?)
', data, re.DOTALL).group(1) + message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags + + match = re.match(".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL) + if match: + ip = match.group(1) + else: + raise Exception("Invalid response: %s" % message) + + if "open" in message: + return {"ip": ip, "opened": True} + elif "closed" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkSubnetonline(self, port): + url = "https://www.subnetonline.com/pages/ipv6-network-tools/online-ipv6-port-scanner.php" + + data = self.requestUrl(url).read() + + ip = re.match('.*Your IP is.*?name="host".*?value="(.*?)"', data, re.DOTALL).group(1) + token = re.match('.*name="token".*?value="(.*?)"', data, re.DOTALL).group(1) + print ip + + post_data = {"host": ip, "port": port, "allow": "on", "token": token, "submit": "Scanning.."} + data = self.requestUrl(url, post_data).read() + + message = re.match(".*
(.*?)
", data, re.DOTALL).group(1) + message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags + + if "online" in message: + return {"ip": ip, "opened": True} + elif "closed" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkMyaddr(self, port): + url = "http://ipv6.my-addr.com/online-ipv6-port-scan.php" + + data = self.requestUrl(url).read() + + ip = re.match('.*Your IP address is:[ ]*([0-9\.:a-z]+)', data.replace(" ", ""), re.DOTALL).group(1) + + post_data = {"addr": ip, "ports_selected": "", "ports_list": port} + data = self.requestUrl(url, post_data).read() + + message = re.match(".*(.*?)
", data, re.DOTALL).group(1) + + if "ok.png" in message: + return {"ip": ip, "opened": True} + elif "fail.png" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkIpv6scanner(self, port): + url = "http://www.ipv6scanner.com/cgi-bin/main.py" + + data = self.requestUrl(url).read() + + ip = re.match('.*Your IP address is[ ]*([0-9\.:a-z]+)', data.replace(" ", ""), re.DOTALL).group(1) + + post_data = {"host": ip, "scanType": "1", "port": port, "protocol": "tcp", "authorized": "yes"} + data = self.requestUrl(url, post_data).read() + + message = re.match(".*(.*?)
", data, re.DOTALL).group(1) + message_text = re.sub("<.*?>", " ", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags + + if "OPEN" in message_text: + return {"ip": ip, "opened": True} + elif "CLOSED" in message_text or "FILTERED" in message_text: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message_text) + +if __name__ == "__main__": + import time + peer_portchecker = PeerPortchecker() + for func_name in ["checkIpv6scanner", "checkMyaddr", "checkPortchecker", "checkCanyouseeme"]: + s = time.time() + print(func_name, getattr(peer_portchecker, func_name)(3894), "%.3fs" % (time.time() - s))