Move port checking to separate file, add ipv6 port check

src/File/FileServer.py

@@ -1,20 +1,17 @@
 import logging
-import urllib2
-import re
 import time
 import random
-import socket
 
 import gevent
 import gevent.pool
 
 import util
+from util import helper
 from Config import config
 from FileRequest import FileRequest
+from Peer import PeerPortchecker
 from Site import SiteManager
-from Debug import Debug
 from Connection import ConnectionServer
-from util import UpnpPunch
 from Plugin import PluginManager
 
 
@@ -23,6 +20,7 @@ class FileServer(ConnectionServer):
 
     def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port):
         self.site_manager = SiteManager.site_manager
+        self.portchecker = PeerPortchecker.PeerPortchecker(self)
         self.log = logging.getLogger("FileServer")
         ip = ip.replace("*", "0.0.0.0")
 
@@ -44,8 +42,7 @@ class FileServer(ConnectionServer):
             self.port_opened = True
             SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
         else:
-            self.port_opened = None  # Is file server opened on router
+            self.port_opened = None  # Check it later
-        self.upnp_port_opened = False
         self.sites = {}
         self.last_request = time.time()
         self.files_parsing = {}
@@ -59,7 +56,7 @@ class FileServer(ConnectionServer):
             if port in tried:
                 continue
             tried.append(port)
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock = helper.createSocket(ip)
             try:
                 sock.bind((ip, port))
                 success = True
@@ -68,6 +65,7 @@ class FileServer(ConnectionServer):
                 success = False
             sock.close()
             if success:
+                self.log.info("Found unused random port: %s" % port)
                 return port
             else:
                 time.sleep(0.1)
@@ -99,157 +97,31 @@ class FileServer(ConnectionServer):
         import imp
         FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest
 
-    # Try to open the port using upnp
-    def openport(self, port=None, check=True):
-        if not port:
-            port = self.port
-        if self.port_opened:
-            return True  # Port already opened
-        if check:  # Check first if its already opened
-            time.sleep(1)  # Wait for port open
-            if self.testOpenport(port, use_alternative=False)["result"] is True:
-                return True  # Port already opened
-
-        if config.tor == "always":  # Port opening won't work in Tor mode
-            return False
-
-        self.log.info("Trying to open port using UpnpPunch...")
-        try:
-            UpnpPunch.ask_to_open_port(self.port, 'ZeroNet', retries=3, protos=["TCP"])
-        except Exception as err:
-            self.log.warning("UpnpPunch run error: %s" % Debug.formatException(err))
-            return False
-
-        if self.testOpenport(port)["result"] is True:
-            self.upnp_port_opened = True
-            return True
-        else:
-            self.log.info("Upnp mapping failed :( Please forward port %s on your router to your ipaddress" % port)
-            return False
-
-    # Test if the port is open
-    def testOpenport(self, port=None, use_alternative=True):
-        if not port:
-            port = self.port
-        back = self.testOpenportPortchecker(port)
-        if (back["result"] is not True and use_alternative) or back["result"] is None:  # If no success try alternative checker
-            back = self.testOpenportCanyouseeme(port)
-
-        if self.ui_server:
-            self.ui_server.updateWebsocket()
-
-        return back
-
-    def testOpenportP2P(self, port=None):
-        self.log.info("Checking port %s using P2P..." % port)
-        site = self.site_manager.get(config.homepage)
-        peers = []
-        res = None
-        if not site:    # First run, has no any peers
-            return self.testOpenportPortchecker(port)  # Fallback to centralized service
-        peers = [peer for peer in site.getRecentPeers(10) if not peer.ip.endswith(".onion")]
-        if len(peers) < 3:   # Not enough peers
-            return self.testOpenportPortchecker(port)  # Fallback to centralized service
-        for retry in range(0, 3):  # Try 3 peers
-            random_peer = random.choice(peers)
-            with gevent.Timeout(10.0, False):  # 10 sec timeout, don't raise exception
-                if not random_peer.connection:
-                    random_peer.connect()
-                if random_peer.connection and random_peer.connection.handshake.get("rev") >= 2186:
-                    res = random_peer.request("checkport", {"port": port})
-                    if res is not None:
-                        break  # All fine, exit from for loop
-
-        if res is None:  # Nobody answered
-            return self.testOpenportPortchecker(port)  # Fallback to centralized service
-        if res["status"] == "closed":
-            if config.tor != "always":
-                self.log.info("[BAD :(] %s says that your port %s is closed" % (random_peer.ip, port))
-            if port == self.port:
-                self.port_opened = False  # Self port, update port_opened status
-                config.ip_external = res["ip_external"]
-                SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
-            return {"result": False}
-        else:
-            self.log.info("[OK :)] %s says that your port %s is open" % (random_peer.ip, port))
-            if port == self.port:  # Self port, update port_opened status
-                self.port_opened = True
-                config.ip_external = res["ip_external"]
-                SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
-            return {"result": True}
-
-    def testOpenportPortchecker(self, port=None):
-        self.log.info("Checking port %s using portchecker.co..." % port)
-        try:
-            data = urllib2.urlopen("https://portchecker.co/check", "port=%s" % port, timeout=20.0).read()
-            message = re.match('.*<div id="results-wrapper">(.*?)</div>', data, re.DOTALL).group(1)
-            message = re.sub("<.*?>", "", message.replace("<br>", " ").replace("&nbsp;", " ").strip())  # Strip http tags
-        except Exception, err:
-            return {"result": None, "message": Debug.formatException(err)}
-
-        if "open" not in message:
-            if config.tor != "always":
-                self.log.info("[BAD :(] Port closed: %s" % message)
-            if port == self.port:
-                self.port_opened = False  # Self port, update port_opened status
-                match = re.match(".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL)  # Try find my external ip in message
-                if match:  # Found my ip in message
-                    config.ip_external = match.group(1)
-                    SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
-                else:
-                    config.ip_external = False
-            return {"result": False, "message": message}
-        else:
-            self.log.info("[OK :)] Port open: %s" % message)
-            if port == self.port:  # Self port, update port_opened status
-                self.port_opened = True
-                match = re.match(".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL)  # Try find my external ip in message
-                if match:  # Found my ip in message
-                    config.ip_external = match.group(1)
-                    SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
-                else:
-                    config.ip_external = False
-            return {"result": True, "message": message}
-
-    def testOpenportCanyouseeme(self, port=None):
-        self.log.info("Checking port %s using canyouseeme.org..." % port)
-        try:
-            data = urllib2.urlopen("http://www.canyouseeme.org/", "port=%s" % port, timeout=20.0).read()
-            message = re.match('.*<p style="padding-left:15px">(.*?)</p>', data, re.DOTALL).group(1)
-            message = re.sub("<.*?>", "", message.replace("<br>", " ").replace("&nbsp;", " "))  # Strip http tags
-        except Exception, err:
-            return {"result": None, "message": Debug.formatException(err)}
-
-        if "Success" not in message:
-            if config.tor != "always":
-                self.log.info("[BAD :(] Port closed: %s" % message)
-            if port == self.port:
-                self.port_opened = False  # Self port, update port_opened status
-                match = re.match(".*?([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)", message)  # Try find my external ip in message
-                if match:  # Found my ip in message
-                    config.ip_external = match.group(1)
-                    SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
-                else:
-                    config.ip_external = False
-            return {"result": False, "message": message}
-        else:
-            self.log.info("[OK :)] Port open: %s" % message)
-            if port == self.port:  # Self port, update port_opened status
-                self.port_opened = True
-                match = re.match(".*?([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)", message)  # Try find my external ip in message
-                if match:  # Found my ip in message
-                    config.ip_external = match.group(1)
-                    SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
-                else:
-                    config.ip_external = False
-            return {"result": True, "message": message}
-
     # Set external ip without testing
     def setIpExternal(self, ip_external):
         logging.info("Setting external ip without testing: %s..." % ip_external)
         config.ip_external = ip_external
         self.port_opened = True
 
+    def portCheck(self):
+        res = self.portchecker.portCheck(self.port, helper.getIpType(self.ip))
+        if not res["opened"]:
+            if self.portchecker.portOpen(self.port):
+                res = self.portchecker.portCheck(self.port, helper.getIpType(self.ip))
+
+        if res["ip"]:
+            config.ip_external = res["ip"]
+            SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
+        else:
+            config.ip_external = False
+
+        if res["opened"]:
+            self.log.info("Server port on %s:%s: Open" % (self.ip, self.port))
+            return True
+        else:
+            self.log.info("Server port on %s:%s: Closed" % (self.ip, self.port))
+            return False
+
     # Check site file integrity
     def checkSite(self, site, check_files=False):
         if site.settings["serving"]:
@@ -272,13 +144,18 @@ class FileServer(ConnectionServer):
 
             if force_port_check:
                 self.port_opened = None
-            self.openport()
+
+            self.port_opened = self.portCheck()
+
             if self.port_opened is False:
                 self.tor_manager.startOnions()
 
         if not sites_checking:
             check_pool = gevent.pool.Pool(5)
-            for site in sorted(self.sites.values(), key=lambda site: site.settings.get("modified", 0), reverse=True):  # Check sites integrity
+            # Check sites integrity
+            for site in sorted(self.sites.values(), key=lambda site: site.settings.get("modified", 0), reverse=True):
+                if not site.settings["serving"]:
+                    continue
                 check_thread = check_pool.spawn(self.checkSite, site, check_files)  # Check in new thread
                 time.sleep(2)
                 if site.settings.get("modified", 0) < time.time() - 60 * 60 * 24:  # Not so active site, wait some sec to finish
@@ -292,7 +169,10 @@ class FileServer(ConnectionServer):
         peers_protected = set([])
         while 1:
             # Sites health care every 20 min
-            self.log.debug("Running site cleanup, connections: %s, internet: %s, protected peers: %s" % (len(self.connections), self.has_internet, peers_protected))
+            self.log.debug(
+                "Running site cleanup, connections: %s, internet: %s, protected peers: %s" %
+                (len(self.connections), self.has_internet, len(peers_protected))
+            )
 
             for address, site in self.sites.items():
                 if not site.settings["serving"]:
@@ -378,7 +258,6 @@ class FileServer(ConnectionServer):
             from Debug import DebugReloader
             DebugReloader(self.reload)
 
-
         if check_sites:  # Open port, Update sites, Check files integrity
             gevent.spawn(self.checkSites)
 
@@ -391,12 +270,12 @@ class FileServer(ConnectionServer):
         self.log.debug("Stopped.")
 
     def stop(self):
-        if self.running and self.upnp_port_opened:
+        if self.running and self.portchecker.upnp_port_opened:
             self.log.debug('Closing port %d' % self.port)
             try:
-                UpnpPunch.ask_to_close_port(self.port, protos=["TCP"])
+                self.portchecker.portClose(self.port)
                 self.log.info('Closed port via upnp.')
-            except (UpnpPunch.UpnpError, UpnpPunch.IGDError), err:
+            except Exception as err:
                 self.log.info("Failed at attempt to use upnp to close port: %s" % err)
 
         return ConnectionServer.stop(self)

src/Peer/PeerPortchecker.py

@@ -0,0 +1,175 @@
+import logging
+import urllib
+import urllib2
+import re
+import time
+
+from Debug import Debug
+from util import UpnpPunch
+
+
+class PeerPortchecker(object):
+    def __init__(self, file_server):
+        self.log = logging.getLogger("PeerPortchecker")
+        self.upnp_port_opened = False
+        self.file_server = file_server
+
+    def requestUrl(self, url, post_data=None):
+        if type(post_data) is dict:
+            post_data = urllib.urlencode(post_data)
+        req = urllib2.Request(url, post_data)
+        req.add_header('Referer', url)
+        return urllib2.urlopen(req, timeout=20.0)
+
+    def portOpen(self, port):
+        self.log.info("Trying to open port using UpnpPunch...")
+
+        try:
+            UpnpPunch.ask_to_open_port(port, 'ZeroNet', retries=3, protos=["TCP"])
+            self.upnp_port_opened = True
+        except Exception as err:
+            self.log.warning("UpnpPunch run error: %s" % Debug.formatException(err))
+            return False
+
+        if self.portCheck(port)["opened"]:
+            return True
+        else:
+            self.log.info("Upnp mapping failed, please forward port %s on your router to your ipaddress" % port)
+            return False
+
+    def portClose(self, port):
+        return UpnpPunch.ask_to_close_port(port, protos=["TCP"])
+
+    def portCheck(self, port, ip_type="ipv4"):
+        if ip_type == "ipv6":
+            checker_functions = ["checkMyaddr", "checkIpv6scanner"]
+        else:
+            checker_functions = ["checkPortchecker", "checkCanyouseeme"]
+
+        for func_name in checker_functions:
+            func = getattr(self, func_name)
+            s = time.time()
+            try:
+                res = func(port)
+                if res:
+                    self.log.info(
+                        "Checking port %s (%s) using %s result: %s in %.3fs" %
+                        (port, ip_type, func_name, res, time.time() - s)
+                    )
+                    time.sleep(0.1)
+                    if  res["opened"] and not self.file_server.had_external_incoming:
+                        res["opened"] = False
+                        self.log.warning("Port %s:%s, but no incoming connection" % (res["ip"], port))
+                    break
+            except Exception as err:
+                self.log.warning(
+                    "%s check error: %s in %.3fs" %
+                    (func_name, Debug.formatException(err), time.time() - s)
+                )
+                res = {"ip": None, "opened": True}
+
+        return res
+
+    def checkCanyouseeme(self, port):
+        data = urllib2.urlopen("http://www.canyouseeme.org/", "port=%s" % port, timeout=20.0).read()
+        message = re.match('.*<p style="padding-left:15px">(.*?)</p>', data, re.DOTALL).group(1)
+        message = re.sub("<.*?>", "", message.replace("<br>", " ").replace("&nbsp;", " "))  # Strip http tags
+
+        match = re.match(".*service on (.*?) on", message)
+        if match:
+            ip = match.group(1)
+        else:
+            raise Exception("Invalid response: %s" % message)
+
+        if "Success" in message:
+            return {"ip": ip, "opened": True}
+        elif "Error" in message:
+            return {"ip": ip, "opened": False}
+        else:
+            raise Exception("Invalid response: %s" % message)
+
+    def checkPortchecker(self, port):
+        data = urllib2.urlopen("https://portchecker.co/check", "port=%s" % port, timeout=20.0).read()
+        message = re.match('.*<div id="results-wrapper">(.*?)</div>', data, re.DOTALL).group(1)
+        message = re.sub("<.*?>", "", message.replace("<br>", " ").replace("&nbsp;", " ").strip())  # Strip http tags
+
+        match = re.match(".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL)
+        if match:
+            ip = match.group(1)
+        else:
+            raise Exception("Invalid response: %s" % message)
+
+        if "open" in message:
+            return {"ip": ip, "opened": True}
+        elif "closed" in message:
+            return {"ip": ip, "opened": False}
+        else:
+            raise Exception("Invalid response: %s" % message)
+
+    def checkSubnetonline(self, port):
+        url = "https://www.subnetonline.com/pages/ipv6-network-tools/online-ipv6-port-scanner.php"
+
+        data = self.requestUrl(url).read()
+
+        ip = re.match('.*Your IP is.*?name="host".*?value="(.*?)"', data, re.DOTALL).group(1)
+        token = re.match('.*name="token".*?value="(.*?)"', data, re.DOTALL).group(1)
+        print ip
+
+        post_data = {"host": ip, "port": port, "allow": "on", "token": token, "submit": "Scanning.."}
+        data = self.requestUrl(url, post_data).read()
+
+        message = re.match(".*<div class='formfield'>(.*?)</div>", data, re.DOTALL).group(1)
+        message = re.sub("<.*?>", "", message.replace("<br>", " ").replace("&nbsp;", " ").strip())  # Strip http tags
+
+        if "online" in message:
+            return {"ip": ip, "opened": True}
+        elif "closed" in message:
+            return {"ip": ip, "opened": False}
+        else:
+            raise Exception("Invalid response: %s" % message)
+
+    def checkMyaddr(self, port):
+        url = "http://ipv6.my-addr.com/online-ipv6-port-scan.php"
+
+        data = self.requestUrl(url).read()
+
+        ip = re.match('.*Your IP address is:[ ]*([0-9\.:a-z]+)', data.replace("&nbsp;", ""), re.DOTALL).group(1)
+
+        post_data = {"addr": ip, "ports_selected": "", "ports_list": port}
+        data = self.requestUrl(url, post_data).read()
+
+        message = re.match(".*<table class='table_font_16'>(.*?)</table>", data, re.DOTALL).group(1)
+
+        if "ok.png" in message:
+            return {"ip": ip, "opened": True}
+        elif "fail.png" in message:
+            return {"ip": ip, "opened": False}
+        else:
+            raise Exception("Invalid response: %s" % message)
+
+    def checkIpv6scanner(self, port):
+        url = "http://www.ipv6scanner.com/cgi-bin/main.py"
+
+        data = self.requestUrl(url).read()
+
+        ip = re.match('.*Your IP address is[ ]*([0-9\.:a-z]+)', data.replace("&nbsp;", ""), re.DOTALL).group(1)
+
+        post_data = {"host": ip, "scanType": "1", "port": port, "protocol": "tcp", "authorized": "yes"}
+        data = self.requestUrl(url, post_data).read()
+
+        message = re.match(".*<table id='scantable'>(.*?)</table>", data, re.DOTALL).group(1)
+        message_text = re.sub("<.*?>", " ", message.replace("<br>", " ").replace("&nbsp;", " ").strip())  # Strip http tags
+
+        if "OPEN" in message_text:
+            return {"ip": ip, "opened": True}
+        elif "CLOSED" in message_text or "FILTERED" in message_text:
+            return {"ip": ip, "opened": False}
+        else:
+            raise Exception("Invalid response: %s" % message_text)
+
+if __name__ == "__main__":
+    import time
+    peer_portchecker = PeerPortchecker()
+    for func_name in ["checkIpv6scanner", "checkMyaddr", "checkPortchecker", "checkCanyouseeme"]:
+        s = time.time()
+        print(func_name, getattr(peer_portchecker, func_name)(3894), "%.3fs" % (time.time() - s))