Rev445, Fix ConnectionServer peer_id handling, Faster startup by creating ssl certs on FileServer start, Per-site connection_server, Fix double Db opening, Test site downloading, Sign testsite properly, Test ConnectionServer, Test FileRequest

.travis.yml

@@ -7,10 +7,10 @@ install:
 before_script:
   - openssl version -a
 script:
- - python -m pytest src/Test --cov src --cov-config src/Test/coverage.ini
+ - python -m pytest src/Test --cov --cov-config src/Test/coverage.ini
 before_install:
+  - pip install -U pytest mock pytest-cov
   - pip install codecov
-  - pip install pytest-cov
   - pip install coveralls
 after_success:
   - codecov

src/Config.py

@@ -8,7 +8,7 @@ class Config(object):
 
     def __init__(self, argv):
         self.version = "0.3.2"
-        self.rev = 431
+        self.rev = 445
         self.argv = argv
         self.action = None
         self.createParser()

src/Connection/Connection.py

@@ -12,7 +12,7 @@ from Crypt import CryptConnection
 
 class Connection(object):
     __slots__ = (
-        "sock", "sock_wrapped", "ip", "port", "peer_id", "id", "protocol", "type", "server", "unpacker", "req_id",
+        "sock", "sock_wrapped", "ip", "port", "id", "protocol", "type", "server", "unpacker", "req_id",
         "handshake", "crypt", "connected", "event_connected", "closed", "start_time", "last_recv_time",
         "last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent",
         "last_ping_delay", "last_req_time", "last_cmd", "name", "updateName", "waiting_requests", "waiting_streams"
@@ -22,7 +22,6 @@ class Connection(object):
         self.sock = sock
         self.ip = ip
         self.port = port
-        self.peer_id = None  # Bittorrent style peer id (not used yet)
         self.id = server.last_connection_id
         server.last_connection_id += 1
         self.protocol = "?"
@@ -161,6 +160,7 @@ class Connection(object):
             self.port = 0
         else:
             self.port = handshake["fileserver_port"]  # Set peer fileserver port
+
         # Check if we can encrypt the connection
         if handshake.get("crypt_supported") and handshake["peer_id"] not in self.server.broken_ssl_peer_ids:
             if handshake.get("crypt"):  # Recommended crypt by server

src/Connection/ConnectionServer.py

@@ -28,7 +28,6 @@ class ConnectionServer:
         self.ip_incoming = {}  # Incoming connections from ip in the last minute to avoid connection flood
         self.broken_ssl_peer_ids = {}  # Peerids of broken ssl connections
         self.ips = {}  # Connection by ip
-        self.peer_ids = {}  # Connections by peer_ids
 
         self.running = True
         self.thread_checker = gevent.spawn(self.checkConnections)
@@ -55,10 +54,9 @@ class ConnectionServer:
             if request_handler:
                 self.handleRequest = request_handler
 
-        CryptConnection.manager.loadCerts()
-
     def start(self):
         self.running = True
+        CryptConnection.manager.loadCerts()
         self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % (
             self.ip, self.port,
             ".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported)
@@ -84,7 +82,7 @@ class ConnectionServer:
                 sock.close()
                 return False
         else:
-            self.ip_incoming[ip] = 0
+            self.ip_incoming[ip] = 1
 
         connection = Connection(self, ip, port, sock)
         self.connections.append(connection)
@@ -92,24 +90,21 @@ class ConnectionServer:
         connection.handleIncomingConnection(sock)
 
     def getConnection(self, ip=None, port=None, peer_id=None, create=True):
-        if peer_id and peer_id in self.peer_ids:  # Find connection by peer id
-            connection = self.peer_ids.get(peer_id)
-            if not connection.connected and create:
-                succ = connection.event_connected.get()  # Wait for connection
-                if not succ:
-                    raise Exception("Connection event return error")
-            return connection
         # Find connection by ip
         if ip in self.ips:
             connection = self.ips[ip]
-            if not connection.connected and create:
+            if not peer_id or connection.handshake.get("peer_id") == peer_id:  # Filter by peer_id
-                succ = connection.event_connected.get()  # Wait for connection
+                if not connection.connected and create:
-                if not succ:
+                    succ = connection.event_connected.get()  # Wait for connection
-                    raise Exception("Connection event return error")
+                    if not succ:
-            return connection
+                        raise Exception("Connection event return error")
+                return connection
+
         # Recover from connection pool
         for connection in self.connections:
             if connection.ip == ip:
+                if peer_id and connection.handshake.get("peer_id") != peer_id:  # Does not match
+                    continue
                 if not connection.connected and create:
                     succ = connection.event_connected.get()  # Wait for connection
                     if not succ:
@@ -141,8 +136,6 @@ class ConnectionServer:
         self.log.debug("Removing %s..." % connection)
         if self.ips.get(connection.ip) == connection:  # Delete if same as in registry
             del self.ips[connection.ip]
-        if connection.peer_id and self.peer_ids.get(connection.peer_id) == connection:  # Delete if same as in registry
-            del self.peer_ids[connection.peer_id]
         if connection in self.connections:
             self.connections.remove(connection)
 

src/Crypt/CryptConnection.py

@@ -53,12 +53,12 @@ class CryptConnectionManager:
         if config.disable_encryption:
             return False
 
-        if self.loadSslRsaCert():
+        if self.createSslRsaCert():
             self.crypt_supported.append("tls-rsa")
 
     # Try to create RSA server cert + sign for connection encryption
     # Return: True on success
-    def loadSslRsaCert(self):
+    def createSslRsaCert(self):
         import subprocess
 
         if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir):
@@ -80,11 +80,11 @@ class CryptConnectionManager:
         if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir):
             return True
         else:
-            logging.error("RSA ECC SSL cert generation failed, cert or key files not exits.")
+            logging.error("RSA ECC SSL cert generation failed, cert or key files not exist.")
             return False
 
     # Not used yet: Missing on some platform
-    def createSslEccCert(self):
+    """def createSslEccCert(self):
         return False
         import subprocess
 
@@ -116,6 +116,6 @@ class CryptConnectionManager:
         else:
             self.logging.error("ECC SSL cert generation failed, cert or key files not exits.")
             return False
-
+    """
 
 manager = CryptConnectionManager()

src/Db/Db.py

@@ -68,9 +68,9 @@ class Db:
         return self.cur.execute(query, params)
 
     def close(self):
+        self.log.debug("Closing, opened: %s" % opened_dbs)
         if self in opened_dbs:
             opened_dbs.remove(self)
-        self.log.debug("Closing")
         if self.cur:
             self.cur.close()
         if self.conn:

src/File/FileServer.py

@@ -15,14 +15,14 @@ from util import UpnpPunch
 
 class FileServer(ConnectionServer):
 
-    def __init__(self):
+    def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port):
-        ConnectionServer.__init__(self, config.fileserver_ip, config.fileserver_port, self.handleRequest)
+        ConnectionServer.__init__(self, ip, port, self.handleRequest)
         if config.ip_external:  # Ip external definied in arguments
             self.port_opened = True
             SiteManager.peer_blacklist.append((config.ip_external, self.port))  # Add myself to peer blacklist
         else:
             self.port_opened = None  # Is file server opened on router
-        self.sites = SiteManager.site_manager.list()
+        self.sites = {}
 
     # Handle request to fileserver
     def handleRequest(self, connection, message):
@@ -228,6 +228,7 @@ class FileServer(ConnectionServer):
 
     # Bind and start serving sites
     def start(self, check_sites=True):
+        self.sites = SiteManager.site_manager.list()
         self.log = logging.getLogger("FileServer")
 
         if config.debug:

src/Peer/Peer.py

@@ -15,7 +15,7 @@ if config.use_tempfiles:
 # Communicate remote peers
 class Peer(object):
     __slots__ = (
-        "ip", "port", "site", "key", "connection_server", "connection", "last_found", "last_response",
+        "ip", "port", "site", "key", "connection", "last_found", "last_response",
         "last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time"
     )
 
@@ -24,7 +24,6 @@ class Peer(object):
         self.port = port
         self.site = site
         self.key = "%s:%s" % (ip, port)
-        self.connection_server = sys.modules["main"].file_server
 
         self.connection = None
         self.last_found = time.time()  # Time of last found in the torrent tracker
@@ -57,7 +56,7 @@ class Peer(object):
             self.connection = None
 
             try:
-                self.connection = self.connection_server.getConnection(self.ip, self.port)
+                self.connection = self.site.connection_server.getConnection(self.ip, self.port)
             except Exception, err:
                 self.onConnectionError()
                 self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
@@ -69,7 +68,7 @@ class Peer(object):
         if self.connection and self.connection.connected:  # We have connection to peer
             return self.connection
         else:  # Try to find from other sites connections
-            self.connection = self.connection_server.getConnection(self.ip, self.port, create=False)
+            self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False)
         return self.connection
 
     def __str__(self):

src/Site/Site.py

@@ -4,7 +4,6 @@ import logging
 import hashlib
 import re
 import time
-import string
 import random
 import sys
 import binascii
@@ -50,7 +49,11 @@ class Site:
         self.storage = SiteStorage(self, allow_create=allow_create)  # Save and load site files
         self.loadSettings()  # Load settings from sites.json
         self.content_manager = ContentManager(self)  # Load contents
-
+        self.connection_server = None
+        if "main" in sys.modules and "file_server" in dir(sys.modules["main"]):  # Use global file server by default if possible
+            self.connection_server = sys.modules["main"].file_server
+        else:
+            self.connection_server = None
         if not self.settings.get("auth_key"):  # To auth user in site (Obsolete, will be removed)
             self.settings["auth_key"] = CryptHash.random()
             self.log.debug("New auth key: %s" % self.settings["auth_key"])
@@ -430,6 +433,7 @@ class Site:
 
         # Rebuild DB
         if new_site.storage.isFile("dbschema.json"):
+            new_site.storage.closeDb()
             new_site.storage.rebuildDb()
 
         return new_site

src/Site/SiteStorage.py

@@ -17,7 +17,6 @@ class SiteStorage:
 
     def __init__(self, site, allow_create=True):
         self.site = site
-        self.fs_encoding = sys.getfilesystemencoding()
         self.directory = "%s/%s" % (config.data_dir, self.site.address)  # Site data diretory
         self.allowed_dir = os.path.abspath(self.directory)  # Only serve/modify file within this dir
         self.log = site.log
@@ -39,7 +38,10 @@ class SiteStorage:
         if check:
             if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:  # Not exist or null
                 self.rebuildDb()
-        self.db = Db(schema, db_path)
+
+        if not self.db:
+            self.db = Db(schema, db_path)
+
         if check and not self.db_checked:
             changed_tables = self.db.checkTables()
             if changed_tables:

src/Test/TestConnection.py

@@ -1,31 +0,0 @@
-import time
-
-from Crypt import CryptConnection
-
-class TestConnection:
-    def testSslConnection(self, connection_server):
-        server = connection_server
-        assert server.running
-
-        # Connect to myself
-        connection = server.getConnection("127.0.0.1", 1544)
-        assert connection.handshake
-        assert connection.crypt
-
-        # Close connection
-        connection.close()
-        time.sleep(0.01)
-        assert len(server.connections) == 0
-
-    def testRawConnection(self, connection_server):
-        server = connection_server
-        crypt_supported_bk = CryptConnection.manager.crypt_supported
-        CryptConnection.manager.crypt_supported = []
-
-        connection = server.getConnection("127.0.0.1", 1544)
-        assert not connection.crypt
-
-        # Close connection
-        connection.close()
-        time.sleep(0.01)
-        assert len(server.connections) == 0

src/Test/TestConnectionServer.py

@@ -0,0 +1,100 @@
+import time
+import gevent
+
+import pytest
+
+from Crypt import CryptConnection
+from Connection import ConnectionServer
+
+
+@pytest.mark.usefixtures("resetSettings")
+class TestConnection:
+    def testSslConnection(self, file_server):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+        assert file_server != client
+
+        # Connect to myself
+        connection = client.getConnection("127.0.0.1", 1544)
+        assert len(file_server.connections) == 1
+        assert len(file_server.ips) == 1
+        assert connection.handshake
+        assert connection.crypt
+
+        # Close connection
+        connection.close()
+        client.stop()
+        time.sleep(0.01)
+        assert len(file_server.connections) == 0
+        assert len(file_server.ips) == 0
+
+    def testRawConnection(self, file_server):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+        assert file_server != client
+
+        # Remove all supported crypto
+        crypt_supported_bk = CryptConnection.manager.crypt_supported
+        CryptConnection.manager.crypt_supported = []
+
+        connection = client.getConnection("127.0.0.1", 1544)
+        assert len(file_server.connections) == 1
+        assert not connection.crypt
+
+        # Close connection
+        connection.close()
+        client.stop()
+        time.sleep(0.01)
+        assert len(file_server.connections) == 0
+
+        # Reset supported crypts
+        CryptConnection.manager.crypt_supported = crypt_supported_bk
+
+    def testPing(self, file_server, site):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+        connection = client.getConnection("127.0.0.1", 1544)
+
+        assert connection.ping()
+
+        connection.close()
+        client.stop()
+
+    def testGetConnection(self, file_server):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+        connection = client.getConnection("127.0.0.1", 1544)
+
+        # Get connection by ip/port
+        connection2 = client.getConnection("127.0.0.1", 1544)
+        assert connection == connection2
+
+        # Get connection by peerid
+        assert not client.getConnection("127.0.0.1", 1544, peer_id="notexists", create=False)
+        connection2 = client.getConnection("127.0.0.1", 1544, peer_id=connection.handshake["peer_id"], create=False)
+        assert connection2 == connection
+
+        connection.close()
+        client.stop()
+
+    def testFloodProtection(self, file_server):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+
+        # Only allow 3 connection in 1 minute
+        connection = client.getConnection("127.0.0.1", 1544)
+        assert connection.handshake
+        connection.close()
+
+        connection = client.getConnection("127.0.0.1", 1544)
+        assert connection.handshake
+        connection.close()
+
+        connection = client.getConnection("127.0.0.1", 1544)
+        assert connection.handshake
+        connection.close()
+
+        # The 4. one will timeout
+        with pytest.raises(gevent.Timeout):
+            with gevent.Timeout(0.1):
+                connection = client.getConnection("127.0.0.1", 1544)

src/Test/TestContent.py

@@ -1,8 +1,11 @@
 import json
+import time
 from cStringIO import StringIO
 
 import pytest
 
+from Crypt import CryptBitcoin
+
 
 @pytest.mark.usefixtures("resetSettings")
 class TestContent:
@@ -31,7 +34,8 @@ class TestContent:
         # Valid signers for root content.json
         assert site.content_manager.getValidSigners("content.json") == ["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"]
 
-    def testSizelimit(self, site):
+    def testLimits(self, site):
+        privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"
         # Data validation
         data_dict = {
             "files": {
@@ -40,29 +44,35 @@ class TestContent:
                     "size": 505
                 }
             },
-            "modified": 1431451896.656619,
+            "modified": time.time()
-            "signs": {
-                "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo":
-                    "G2QC+ZIozPQQ/XiOEOMzfekOP8ipi+rKaTy/R/3MnDf98mLIhSSA8927FW6D/ZyP7HHuII2y9d0zbAk+rr8ksQM="
-            }
         }
-        data = StringIO(json.dumps(data_dict))
 
         # Normal data
+        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) }
+        data = StringIO(json.dumps(data_dict))
         assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
+        # Reset
+        del data_dict["signs"]
 
         # Too large
         data_dict["files"]["data.json"]["size"] = 200000  # Emulate 2MB sized data.json
+        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) }
         data = StringIO(json.dumps(data_dict))
         assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
-        data_dict["files"]["data.json"]["size"] = 505  # Reset
+        # Reset
+        data_dict["files"]["data.json"]["size"] = 505
+        del data_dict["signs"]
 
         # Not allowed file
         data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"]
+        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) }
         data = StringIO(json.dumps(data_dict))
         assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
-        del data_dict["files"]["notallowed.exe"]  # Reset
+        # Reset
+        del data_dict["files"]["notallowed.exe"]
+        del data_dict["signs"]
 
         # Should work again
+        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) }
         data = StringIO(json.dumps(data_dict))
         assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)

src/Test/TestCryptConnection.py

@@ -21,7 +21,3 @@ class TestCryptConnection:
         # Check openssl cert generation
         assert os.path.isfile("%s/cert-rsa.pem" % config.data_dir)
         assert os.path.isfile("%s/key-rsa.pem" % config.data_dir)
-
-        # Remove created files
-        os.unlink("%s/cert-rsa.pem" % config.data_dir)
-        os.unlink("%s/key-rsa.pem" % config.data_dir)

src/Test/TestEvent.py

@@ -1,7 +1,7 @@
 import util
 
 
-class TestClass(object):
+class ExampleClass(object):
     def __init__(self):
         self.called = []
         self.onChanged = util.Event()
@@ -12,7 +12,7 @@ class TestClass(object):
 
 class TestEvent:
     def testEvent(self):
-        test_obj = TestClass()
+        test_obj = ExampleClass()
         test_obj.onChanged.append(lambda: test_obj.increment("Called #1"))
         test_obj.onChanged.append(lambda: test_obj.increment("Called #2"))
         test_obj.onChanged.once(lambda: test_obj.increment("Once"))
@@ -25,7 +25,7 @@ class TestEvent:
         assert test_obj.called == ["Called #1", "Called #2", "Once", "Called #1", "Called #2", "Called #1", "Called #2"]
 
     def testOnce(self):
-        test_obj = TestClass()
+        test_obj = ExampleClass()
         test_obj.onChanged.once(lambda: test_obj.increment("Once test #1"))
 
         # It should be called only once
@@ -37,7 +37,7 @@ class TestEvent:
         assert test_obj.called == ["Once test #1"]
 
     def testOnceMultiple(self):
-        test_obj = TestClass()
+        test_obj = ExampleClass()
         # Allow queue more than once
         test_obj.onChanged.once(lambda: test_obj.increment("Once test #1"))
         test_obj.onChanged.once(lambda: test_obj.increment("Once test #2"))
@@ -51,7 +51,7 @@ class TestEvent:
         assert test_obj.called == ["Once test #1", "Once test #2", "Once test #3"]
 
     def testOnceNamed(self):
-        test_obj = TestClass()
+        test_obj = ExampleClass()
         # Dont store more that one from same type
         test_obj.onChanged.once(lambda: test_obj.increment("Once test #1/1"), "type 1")
         test_obj.onChanged.once(lambda: test_obj.increment("Once test #1/2"), "type 1")

src/Test/TestFileRequest.py

@@ -0,0 +1,35 @@
+import cStringIO as StringIO
+
+import pytest
+
+from Connection import ConnectionServer
+
+
+@pytest.mark.usefixtures("resetSettings")
+class TestFileRequest:
+    def testGetFile(self, file_server, site):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+
+        connection = client.getConnection("127.0.0.1", 1544)
+        file_server.sites[site.address] = site
+
+        response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0})
+        assert "sign" in response["body"]
+
+        connection.close()
+        client.stop()
+
+    def testStreamFile(self, file_server, site):
+        file_server.ip_incoming = {}  # Reset flood protection
+        client = ConnectionServer("127.0.0.1", 1545)
+        connection = client.getConnection("127.0.0.1", 1544)
+        file_server.sites[site.address] = site
+
+        buff = StringIO.StringIO()
+        response = connection.request("streamFile", {"site": site.address, "inner_path": "content.json", "location": 0}, buff)
+        assert "stream_bytes" in response
+        assert "sign" in buff.getvalue()
+
+        connection.close()
+        client.stop()

src/Test/TestNoparallel.py

@@ -6,7 +6,7 @@ monkey.patch_all()
 
 import util
 
-class TestClass(object):
+class ExampleClass(object):
     def __init__(self):
         self.counted = 0
 
@@ -27,8 +27,8 @@ class TestClass(object):
 
 class TestNoparallel:
     def testBlocking(self):
-        obj1 = TestClass()
+        obj1 = ExampleClass()
-        obj2 = TestClass()
+        obj2 = ExampleClass()
 
         # Dont allow to call again until its running and wait until its running
         threads = [
@@ -46,8 +46,8 @@ class TestNoparallel:
         assert obj2.counted == 10
 
     def testNoblocking(self):
-        obj1 = TestClass()
+        obj1 = ExampleClass()
-        obj2 = TestClass()
+        obj2 = ExampleClass()
 
         thread1 = obj1.countNoblocking()
         thread2 = obj1.countNoblocking()  # Ignored

src/Test/TestRateLimit.py

@@ -7,12 +7,12 @@ monkey.patch_all()
 from util import RateLimit
 
 
-# Time is around limit +/- 0.01 sec
+# Time is around limit +/- 0.05 sec
 def around(t, limit):
-    return t >= limit - 0.01 and t <= limit + 0.01
+    return t >= limit - 0.05 and t <= limit + 0.05
 
 
-class TestClass(object):
+class ExampleClass(object):
     def __init__(self):
         self.counted = 0
         self.last_called = None
@@ -25,8 +25,8 @@ class TestClass(object):
 
 class TestRateLimit:
     def testCall(self):
-        obj1 = TestClass()
+        obj1 = ExampleClass()
-        obj2 = TestClass()
+        obj2 = ExampleClass()
 
         s = time.time()
         assert RateLimit.call("counting", allowed_again=0.1, func=obj1.count) == "counted"
@@ -61,8 +61,8 @@ class TestRateLimit:
         assert obj2.counted == 4
 
     def testCallAsync(self):
-        obj1 = TestClass()
+        obj1 = ExampleClass()
-        obj2 = TestClass()
+        obj2 = ExampleClass()
 
         s = time.time()
         RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #1").join()

src/Test/TestWorker.py

@@ -0,0 +1,35 @@
+import time
+import os
+
+import gevent
+import pytest
+import mock
+
+from Crypt import CryptConnection
+from Connection import ConnectionServer
+from Config import config
+from Site import Site
+
+@pytest.mark.usefixtures("resetTempSettings")
+@pytest.mark.usefixtures("resetSettings")
+class TestWorker:
+    def testDownload(self, file_server, site, site_temp):
+        client = ConnectionServer("127.0.0.1", 1545)
+        assert site.storage.directory == config.data_dir+"/"+site.address
+        assert site_temp.storage.directory == config.data_dir+"-temp/"+site.address
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        site_temp.connection_server = client
+        site_temp.announce = mock.MagicMock(return_value=True)  # Don't try to find peers from the net
+
+        # Download to client from source
+        site_temp.addPeer("127.0.0.1", 1544)
+        site_temp.download().join(timeout=5)
+
+        assert not site_temp.bad_files
+
+        site_temp.storage.deleteFiles()

src/Test/conftest.py

@@ -2,8 +2,10 @@ import os
 import sys
 import urllib
 import time
+import logging
 
 import pytest
+import mock
 
 # Config
 if sys.platform == "win32":
@@ -14,24 +16,27 @@ SITE_URL = "http://127.0.0.1:43110"
 
 # Imports relative to src dir
 sys.path.append(
-    os.path.abspath(os.path.dirname(__file__)+"/..")
+    os.path.abspath(os.path.dirname(__file__) + "/..")
 )
 from Config import config
 config.argv = ["none"]  # Dont pass any argv to config parser
 config.parse()
 config.data_dir = "src/Test/testdata"  # Use test data for unittests
+logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
 
 from Site import Site
 from User import UserManager
+from File import FileServer
 from Connection import ConnectionServer
 from Crypt import CryptConnection
 import gevent
 from gevent import monkey
 monkey.patch_all(thread=False)
 
+
 @pytest.fixture(scope="session")
 def resetSettings(request):
-    os.chdir(os.path.abspath(os.path.dirname(__file__)+"/../..")) # Set working dir
+    os.chdir(os.path.abspath(os.path.dirname(__file__) + "/../.."))  # Set working dir
     open("%s/sites.json" % config.data_dir, "w").write("{}")
     open("%s/users.json" % config.data_dir, "w").write("""
         {
@@ -42,23 +47,58 @@ def resetSettings(request):
             }
         }
     """)
+
     def cleanup():
         os.unlink("%s/sites.json" % config.data_dir)
         os.unlink("%s/users.json" % config.data_dir)
     request.addfinalizer(cleanup)
 
+
+@pytest.fixture(scope="session")
+def resetTempSettings(request):
+    data_dir_temp = config.data_dir + "-temp"
+    if not os.path.isdir(data_dir_temp):
+        os.mkdir(data_dir_temp)
+    open("%s/sites.json" % data_dir_temp, "w").write("{}")
+    open("%s/users.json" % data_dir_temp, "w").write("""
+        {
+            "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc": {
+                "certs": {},
+                "master_seed": "024bceac1105483d66585d8a60eaf20aa8c3254b0f266e0d626ddb6114e2949a",
+                "sites": {}
+            }
+        }
+    """)
+
+    def cleanup():
+        os.unlink("%s/sites.json" % data_dir_temp)
+        os.unlink("%s/users.json" % data_dir_temp)
+    request.addfinalizer(cleanup)
+
+
 @pytest.fixture(scope="session")
 def site():
     site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
     return site
 
 
+@pytest.fixture()
+def site_temp(request):
+    with mock.patch("Config.config.data_dir", config.data_dir+"-temp"):
+        site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
+    def cleanup():
+        site_temp.storage.deleteFiles()
+    request.addfinalizer(cleanup)
+    return site_temp
+
+
 @pytest.fixture(scope="session")
 def user():
     user = UserManager.user_manager.get()
     user.sites = {}  # Reset user data
     return user
 
+
 @pytest.fixture(scope="session")
 def browser():
     try:
@@ -69,6 +109,7 @@ def browser():
         raise pytest.skip("Test requires selenium + phantomjs: %s" % err)
     return browser
 
+
 @pytest.fixture(scope="session")
 def site_url():
     try:
@@ -77,9 +118,14 @@ def site_url():
         raise pytest.skip("Test requires zeronet client running: %s" % err)
     return SITE_URL
 
+
 @pytest.fixture(scope="session")
-def connection_server():
+def file_server(request):
-    connection_server = ConnectionServer("127.0.0.1", 1544)
+    CryptConnection.manager.loadCerts()  # Load and create certs
-    gevent.spawn(connection_server.start)
+    request.addfinalizer(CryptConnection.manager.removeCerts)  # Remove cert files after end
-    time.sleep(0) # Port opening
+    file_server = FileServer("127.0.0.1", 1544)
-    return connection_server
+    gevent.spawn(lambda: ConnectionServer.start(file_server))
+    time.sleep(0)  # Port opening
+    assert file_server.running
+    return file_server
+

src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json

@@ -1,129 +1,133 @@
 {
-  "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", 
+ "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", 
-  "background-color": "white", 
+ "background-color": "white", 
-  "description": "Blogging platform Demo", 
+ "description": "Blogging platform Demo", 
-  "domain": "Blog.ZeroNetwork.bit", 
+ "domain": "Blog.ZeroNetwork.bit", 
-  "files": {
+ "files": {
-    "css/all.css": {
+  "css/all.css": {
-      "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", 
+   "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", 
-      "size": 112710
+   "size": 112710
-    }, 
-    "data-default/data.json": {
-      "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", 
-      "size": 196
-    }, 
-    "data-default/users/content-default.json": {
-      "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", 
-      "size": 679
-    }, 
-    "data/data.json": {
-      "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", 
-      "size": 31126
-    }, 
-    "data/img/autoupdate.png": {
-      "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", 
-      "size": 24460
-    }, 
-    "data/img/direct_domains.png": {
-      "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", 
-      "size": 16185
-    }, 
-    "data/img/domain.png": {
-      "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", 
-      "size": 11881
-    }, 
-    "data/img/memory.png": {
-      "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", 
-      "size": 12775
-    }, 
-    "data/img/multiuser.png": {
-      "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", 
-      "size": 29480
-    }, 
-    "data/img/progressbar.png": {
-      "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", 
-      "size": 13294
-    }, 
-    "data/img/slides.png": {
-      "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", 
-      "size": 14439
-    }, 
-    "data/img/slots_memory.png": {
-      "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", 
-      "size": 9488
-    }, 
-    "data/img/trayicon.png": {
-      "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", 
-      "size": 19040
-    }, 
-    "data/img/zeroblog-comments.png": {
-      "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", 
-      "size": 24001
-    }, 
-    "data/img/zeroid.png": {
-      "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", 
-      "size": 18875
-    }, 
-    "data/img/zeroname.png": {
-      "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", 
-      "size": 36031
-    }, 
-    "data/img/zerotalk-mark.png": {
-      "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", 
-      "size": 44862
-    }, 
-    "data/img/zerotalk-upvote.png": {
-      "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", 
-      "size": 41092
-    }, 
-    "data/img/zerotalk.png": {
-      "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", 
-      "size": 26606
-    }, 
-    "dbschema.json": {
-      "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", 
-      "size": 1529
-    }, 
-    "img/loading.gif": {
-      "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", 
-      "size": 723
-    }, 
-    "index.html": {
-      "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", 
-      "size": 4667
-    }, 
-    "js/all.js": {
-      "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", 
-      "size": 201178
-    }
   }, 
-  "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", 
+  "data-default/data.json": {
-  "includes": {
+   "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", 
-    "data/users/content.json": {
+   "size": 196
-      "signers": ["1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f"], 
-      "signers_required": 1
-    },
-    "data/test_include/content.json": {
-      "added": 1424976057, 
-      "files_allowed": "data.json", 
-      "includes_allowed": false, 
-      "max_size": 20000, 
-      "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], 
-      "signers_required": 1, 
-      "user_id": 47, 
-      "user_name": "test"
-    }
   }, 
-  "modified": 1434801613.8, 
+  "data-default/users/content-default.json": {
-  "sign": [
+   "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", 
-    107584248894581661953399064048991976924739126704981340547735906786807630121376, 
+   "size": 679
-    14052922268999375798453683972186312380248481029778104103759595432459320456230
-  ], 
-  "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", 
-  "signs": {
-    "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HO9esgmZhPJqOG+fEpStwK+u5P/+4Kx5VGApNnbsyA0lBfEV6aWviIP6FBlP3sZSH/EMuoEw42lToRmLppbzmuM="
   }, 
-  "signs_required": 1, 
+  "data/data.json": {
-  "title": "ZeroBlog", 
+   "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", 
-  "zeronet_version": "0.3.1"
+   "size": 31126
+  }, 
+  "data/img/autoupdate.png": {
+   "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", 
+   "size": 24460
+  }, 
+  "data/img/direct_domains.png": {
+   "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", 
+   "size": 16185
+  }, 
+  "data/img/domain.png": {
+   "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", 
+   "size": 11881
+  }, 
+  "data/img/memory.png": {
+   "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", 
+   "size": 12775
+  }, 
+  "data/img/multiuser.png": {
+   "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", 
+   "size": 29480
+  }, 
+  "data/img/progressbar.png": {
+   "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", 
+   "size": 13294
+  }, 
+  "data/img/slides.png": {
+   "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", 
+   "size": 14439
+  }, 
+  "data/img/slots_memory.png": {
+   "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", 
+   "size": 9488
+  }, 
+  "data/img/trayicon.png": {
+   "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", 
+   "size": 19040
+  }, 
+  "data/img/zeroblog-comments.png": {
+   "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", 
+   "size": 24001
+  }, 
+  "data/img/zeroid.png": {
+   "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", 
+   "size": 18875
+  }, 
+  "data/img/zeroname.png": {
+   "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", 
+   "size": 36031
+  }, 
+  "data/img/zerotalk-mark.png": {
+   "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", 
+   "size": 44862
+  }, 
+  "data/img/zerotalk-upvote.png": {
+   "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", 
+   "size": 41092
+  }, 
+  "data/img/zerotalk.png": {
+   "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", 
+   "size": 26606
+  }, 
+  "data/test_include/data.json": {
+   "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", 
+   "size": 505
+  }, 
+  "dbschema.json": {
+   "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", 
+   "size": 1529
+  }, 
+  "img/loading.gif": {
+   "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", 
+   "size": 723
+  }, 
+  "index.html": {
+   "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", 
+   "size": 4667
+  }, 
+  "js/all.js": {
+   "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", 
+   "size": 201178
+  }
+ }, 
+ "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", 
+ "includes": {
+  "data/test_include/content.json": {
+   "added": 1424976057, 
+   "files_allowed": "data.json", 
+   "includes_allowed": false, 
+   "max_size": 20000, 
+   "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], 
+   "signers_required": 1, 
+   "user_id": 47, 
+   "user_name": "test"
+  }, 
+  "data/users/content.json": {
+   "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], 
+   "signers_required": 1
+  }
+ }, 
+ "modified": 1443088239.123, 
+ "sign": [
+  37796247323133993908968541760020085519225012317332056166386012116450888757672, 
+  8182016604193300184892407269063757269964429504791487428802219119125679030316
+ ], 
+ "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", 
+ "signs": {
+  "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HJ+SuvmYh1DIyvqypUobaspZ3heUfYWoN34S4c2la5NgcBmpZ/YN4Xzi6wtP20W8DePXdsYMC0Azr+L8ZF7FAk4="
+ }, 
+ "signs_required": 1, 
+ "title": "ZeroBlog", 
+ "zeronet_version": "0.3.2"
 }

src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/test_include/content.json

@@ -1,12 +1,12 @@
 {
-  "files": {
+ "files": {
-    "data.json": {
+  "data.json": {
-      "sha512": "f6ea25af270ba6a67c90a053c34da8ba94e6cf2177d4ee7979fd517a31ca6479", 
+   "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", 
-      "size": 74
+   "size": 505
-    }
-  }, 
-  "modified": 1424976057.772182, 
-  "signs": {
-    "1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ": "G1Jy36d3LLu+Lh8ikGqOozyiYZ+NvF8QF1OdC6PfDt26bflPPQ0gwWw8AQdFmc/S3BMBDNt0tJshiiJcRK46j/c="
   }
+ }, 
+ "modified": 1443088412.024, 
+ "signs": {
+  "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HPpRa/7ic/03aJ6vfz3zt3ezsnkDeaet85HGS3Rm9vCXWGsdOXboMynb/sZcTfPMC1bQ3zLRdUNMqmifKw/gnNg="
+ }
 }

src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json

@@ -1,26 +1,26 @@
 {
-  "files": {}, 
+ "files": {}, 
-  "ignore": ".*", 
+ "ignore": ".*", 
-  "modified": 1432466966.003, 
+ "modified": 1443088330.941, 
-  "signs": {
+ "signs": {
-    "1BLogC9LN4oPDcruNz3qo1ysa133E9AGg8": "HChU28lG4MCnAiui6wDAaVCD4QUrgSy4zZ67+MMHidcUJRkLGnO3j4Eb1N0AWQ86nhSBwoOQf08Rha7gRyTDlAk="
+  "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G/YCfchtojDA7EjXk5Xa6af5EaEME14LDAvVE9P8PCDb2ncWN79ZTMsczAx7N3HYyM9Vdqn+8or4hh28z4ITKqU="
+ }, 
+ "user_contents": {
+  "cert_signers": {
+   "zeroid.bit": [ "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ]
   }, 
-  "user_contents": {
+  "permission_rules": {
-    "cert_signers": {
+   ".*": {
-      "zeroid.bit": [ "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ]
+    "files_allowed": "data.json", 
-    }, 
+    "max_size": 10000, 
-    "permission_rules": {
+    "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ]
-      ".*": {
+   }, 
-        "files_allowed": "data.json", 
+   "bitid/.*@zeroid.bit": { "max_size": 40000 }, 
-        "max_size": 10000,
+   "bitmsg/.*@zeroid.bit": { "max_size": 15000 }
-        "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ]
+  }, 
-      }, 
+  "permissions": {
-      "bitid/.*@zeroid.bit": { "max_size": 40000 }, 
+   "bad@zeroid.bit": false, 
-      "bitmsg/.*@zeroid.bit": { "max_size": 15000 }
+   "nofish@zeroid.bit": { "max_size": 100000 }
-    }, 
-    "permissions": {
-      "bad@zeroid.bit": false, 
-      "nofish@zeroid.bit": { "max_size": 100000 }
-    }
   }
+ }
 }

src/main.py

@@ -96,6 +96,8 @@ class Actions(object):
         global ui_server, file_server
         from File import FileServer
         from Ui import UiServer
+        logging.info("Creating FileServer....")
+        file_server = FileServer()
         logging.info("Creating UiServer....")
         ui_server = UiServer()
 
@@ -103,9 +105,6 @@ class Actions(object):
         from Crypt import CryptConnection
         CryptConnection.manager.removeCerts()
 
-        logging.info("Creating FileServer....")
-        file_server = FileServer()
-
         logging.info("Starting servers....")
         gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)])