Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
This commit is contained in:
HelloZeroNet
parent
c9578e9037
commit
e9d2cdfd37
99 changed files with 9476 additions and 267 deletions
src/Connection
|
|
@ -1,5 +1,6 @@
|
|||
import socket
|
||||
import time
|
||||
import hashlib
|
||||
|
||||
import gevent
|
||||
import msgpack
|
||||
|
|
@ -8,20 +9,25 @@ from Config import config
|
|||
from Debug import Debug
|
||||
from util import StreamingMsgpack
|
||||
from Crypt import CryptConnection
|
||||
from Site import SiteManager
|
||||
|
||||
|
||||
class Connection(object):
|
||||
__slots__ = (
|
||||
"sock", "sock_wrapped", "ip", "port", "id", "protocol", "type", "server", "unpacker", "req_id",
|
||||
"sock", "sock_wrapped", "ip", "port", "cert_pin", "site_lock", "id", "protocol", "type", "server", "unpacker", "req_id",
|
||||
"handshake", "crypt", "connected", "event_connected", "closed", "start_time", "last_recv_time",
|
||||
"last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent",
|
||||
"last_ping_delay", "last_req_time", "last_cmd", "name", "updateName", "waiting_requests", "waiting_streams"
|
||||
)
|
||||
|
||||
def __init__(self, server, ip, port, sock=None):
|
||||
def __init__(self, server, ip, port, sock=None, site_lock=None):
|
||||
self.sock = sock
|
||||
self.ip = ip
|
||||
self.port = port
|
||||
self.cert_pin = None
|
||||
if "#" in ip:
|
||||
self.ip, self.cert_pin = ip.split("#")
|
||||
self.site_lock = site_lock # Only this site requests allowed (for Tor)
|
||||
self.id = server.last_connection_id
|
||||
server.last_connection_id += 1
|
||||
self.protocol = "?"
|
||||
|
|
@ -73,17 +79,23 @@ class Connection(object):
|
|||
def connect(self):
|
||||
self.log("Connecting...")
|
||||
self.type = "out"
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.sock.connect((self.ip, int(self.port)))
|
||||
if self.ip.endswith(".onion"):
|
||||
if not self.server.tor_manager or not self.server.tor_manager.enabled:
|
||||
raise Exception("Can't connect to onion addresses, no Tor controller present")
|
||||
self.sock = self.server.tor_manager.createSocket(self.ip, self.port)
|
||||
else:
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.sock.connect((self.ip, int(self.port)))
|
||||
|
||||
# Implicit SSL in the future
|
||||
# self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa")
|
||||
# self.sock.do_handshake()
|
||||
# self.crypt = "tls-rsa"
|
||||
# self.sock_wrapped = True
|
||||
# Implicit SSL
|
||||
if self.cert_pin:
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", cert_pin=self.cert_pin)
|
||||
self.sock.do_handshake()
|
||||
self.crypt = "tls-rsa"
|
||||
self.sock_wrapped = True
|
||||
|
||||
# Detect protocol
|
||||
self.send({"cmd": "handshake", "req_id": 0, "params": self.handshakeInfo()})
|
||||
self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()})
|
||||
event_connected = self.event_connected
|
||||
gevent.spawn(self.messageLoop)
|
||||
return event_connected.get() # Wait for handshake
|
||||
|
|
@ -92,14 +104,15 @@ class Connection(object):
|
|||
def handleIncomingConnection(self, sock):
|
||||
self.log("Incoming connection...")
|
||||
self.type = "in"
|
||||
try:
|
||||
if sock.recv(1, gevent.socket.MSG_PEEK) == "\x16":
|
||||
self.log("Crypt in connection using implicit SSL")
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", True)
|
||||
self.sock_wrapped = True
|
||||
self.crypt = "tls-rsa"
|
||||
except Exception, err:
|
||||
self.log("Socket peek error: %s" % Debug.formatException(err))
|
||||
if self.ip != "127.0.0.1": # Clearnet: Check implicit SSL
|
||||
try:
|
||||
if sock.recv(1, gevent.socket.MSG_PEEK) == "\x16":
|
||||
self.log("Crypt in connection using implicit SSL")
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", True)
|
||||
self.sock_wrapped = True
|
||||
self.crypt = "tls-rsa"
|
||||
except Exception, err:
|
||||
self.log("Socket peek error: %s" % Debug.formatException(err))
|
||||
self.messageLoop()
|
||||
|
||||
# Message loop for connection
|
||||
|
|
@ -142,28 +155,60 @@ class Connection(object):
|
|||
self.close() # MessageLoop ended, close connection
|
||||
|
||||
# My handshake info
|
||||
def handshakeInfo(self):
|
||||
return {
|
||||
def getHandshakeInfo(self):
|
||||
# No TLS for onion connections
|
||||
if self.ip.endswith(".onion"):
|
||||
crypt_supported = []
|
||||
else:
|
||||
crypt_supported = CryptConnection.manager.crypt_supported
|
||||
# No peer id for onion connections
|
||||
if self.ip.endswith(".onion") or self.ip == "127.0.0.1":
|
||||
peer_id = ""
|
||||
else:
|
||||
peer_id = self.server.peer_id
|
||||
# Setup peer lock from requested onion address
|
||||
if self.handshake and self.handshake.get("target_ip", "").endswith(".onion"):
|
||||
target_onion = self.handshake.get("target_ip").replace(".onion", "") # My onion address
|
||||
onion_sites = {v: k for k, v in self.server.tor_manager.site_onions.items()} # Inverse, Onion: Site address
|
||||
self.site_lock = onion_sites.get(target_onion)
|
||||
if not self.site_lock:
|
||||
self.server.log.error("Unknown target onion address: %s" % target_onion)
|
||||
self.site_lock = "unknown"
|
||||
|
||||
handshake = {
|
||||
"version": config.version,
|
||||
"protocol": "v2",
|
||||
"peer_id": self.server.peer_id,
|
||||
"peer_id": peer_id,
|
||||
"fileserver_port": self.server.port,
|
||||
"port_opened": self.server.port_opened,
|
||||
"target_ip": self.ip,
|
||||
"rev": config.rev,
|
||||
"crypt_supported": CryptConnection.manager.crypt_supported,
|
||||
"crypt_supported": crypt_supported,
|
||||
"crypt": self.crypt
|
||||
}
|
||||
if self.site_lock:
|
||||
handshake["onion"] = self.server.tor_manager.getOnion(self.site_lock)
|
||||
elif self.ip.endswith(".onion"):
|
||||
handshake["onion"] = self.server.tor_manager.getOnion("global")
|
||||
|
||||
return handshake
|
||||
|
||||
def setHandshake(self, handshake):
|
||||
self.handshake = handshake
|
||||
if handshake.get("port_opened", None) is False: # Not connectable
|
||||
if handshake.get("port_opened", None) is False and not "onion" in handshake: # Not connectable
|
||||
self.port = 0
|
||||
else:
|
||||
self.port = handshake["fileserver_port"] # Set peer fileserver port
|
||||
|
||||
if handshake.get("onion") and not self.ip.endswith(".onion"): # Set incoming connection's onion address
|
||||
self.ip = handshake["onion"] + ".onion"
|
||||
self.updateName()
|
||||
|
||||
# Check if we can encrypt the connection
|
||||
if handshake.get("crypt_supported") and handshake["peer_id"] not in self.server.broken_ssl_peer_ids:
|
||||
if handshake.get("crypt"): # Recommended crypt by server
|
||||
if self.ip.endswith(".onion"):
|
||||
crypt = None
|
||||
elif handshake.get("crypt"): # Recommended crypt by server
|
||||
crypt = handshake["crypt"]
|
||||
else: # Select the best supported on both sides
|
||||
crypt = CryptConnection.manager.selectCrypt(handshake["crypt_supported"])
|
||||
|
|
@ -193,30 +238,21 @@ class Connection(object):
|
|||
self.crypt = message["crypt"]
|
||||
server = (self.type == "in")
|
||||
self.log("Crypt out connection using: %s (server side: %s)..." % (self.crypt, server))
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server)
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin)
|
||||
self.sock.do_handshake()
|
||||
self.sock_wrapped = True
|
||||
|
||||
if not self.sock_wrapped and self.cert_pin:
|
||||
self.log("Crypt connection error: Socket not encrypted, but certificate pin present")
|
||||
self.close()
|
||||
return
|
||||
|
||||
self.setHandshake(message)
|
||||
else:
|
||||
self.log("Unknown response: %s" % message)
|
||||
elif message.get("cmd"): # Handhsake request
|
||||
if message["cmd"] == "handshake":
|
||||
if config.debug_socket:
|
||||
self.log("Handshake request: %s" % message)
|
||||
self.setHandshake(message["params"])
|
||||
data = self.handshakeInfo()
|
||||
data["cmd"] = "response"
|
||||
data["to"] = message["req_id"]
|
||||
self.send(data) # Send response to handshake
|
||||
# Sent crypt request to client
|
||||
if self.crypt and not self.sock_wrapped:
|
||||
server = (self.type == "in")
|
||||
self.log("Crypt in connection using: %s (server side: %s)..." % (self.crypt, server))
|
||||
try:
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server)
|
||||
self.sock_wrapped = True
|
||||
except Exception, err:
|
||||
self.log("Crypt connection error: %s, adding peerid %s as broken ssl." % (err, message["params"]["peer_id"]))
|
||||
self.server.broken_ssl_peer_ids[message["params"]["peer_id"]] = True
|
||||
self.handleHandshake(message)
|
||||
else:
|
||||
self.server.handleRequest(self, message)
|
||||
else: # Old style response, no req_id definied
|
||||
|
|
@ -226,6 +262,30 @@ class Connection(object):
|
|||
self.waiting_requests[last_req_id].set(message)
|
||||
del self.waiting_requests[last_req_id] # Remove from waiting request
|
||||
|
||||
# Incoming handshake set request
|
||||
def handleHandshake(self, message):
|
||||
if config.debug_socket:
|
||||
self.log("Handshake request: %s" % message)
|
||||
self.setHandshake(message["params"])
|
||||
data = self.getHandshakeInfo()
|
||||
data["cmd"] = "response"
|
||||
data["to"] = message["req_id"]
|
||||
self.send(data) # Send response to handshake
|
||||
# Sent crypt request to client
|
||||
if self.crypt and not self.sock_wrapped:
|
||||
server = (self.type == "in")
|
||||
self.log("Crypt in connection using: %s (server side: %s)..." % (self.crypt, server))
|
||||
try:
|
||||
self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin)
|
||||
self.sock_wrapped = True
|
||||
except Exception, err:
|
||||
self.log("Crypt connection error: %s, adding peerid %s as broken ssl." % (err, message["params"]["peer_id"]))
|
||||
self.server.broken_ssl_peer_ids[message["params"]["peer_id"]] = True
|
||||
|
||||
if not self.sock_wrapped and self.cert_pin:
|
||||
self.log("Crypt connection error: Socket not encrypted, but certificate pin present")
|
||||
self.close()
|
||||
|
||||
# Stream socket directly to a file
|
||||
def handleStream(self, message):
|
||||
if config.debug_socket:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
import logging
|
||||
import random
|
||||
import string
|
||||
import time
|
||||
import sys
|
||||
|
||||
|
|
@ -14,6 +12,7 @@ from Connection import Connection
|
|||
from Config import config
|
||||
from Crypt import CryptConnection
|
||||
from Crypt import CryptHash
|
||||
from Tor import TorManager
|
||||
|
||||
|
||||
class ConnectionServer:
|
||||
|
|
@ -24,7 +23,13 @@ class ConnectionServer:
|
|||
self.log = logging.getLogger("ConnServer")
|
||||
self.port_opened = None
|
||||
|
||||
if config.tor != "disabled":
|
||||
self.tor_manager = TorManager(self.ip, self.port)
|
||||
else:
|
||||
self.tor_manager = None
|
||||
|
||||
self.connections = [] # Connections
|
||||
self.whitelist = ("127.0.0.1",) # No flood protection on this ips
|
||||
self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood
|
||||
self.broken_ssl_peer_ids = {} # Peerids of broken ssl connections
|
||||
self.ips = {} # Connection by ip
|
||||
|
|
@ -41,7 +46,7 @@ class ConnectionServer:
|
|||
# Check msgpack version
|
||||
if msgpack.version[0] == 0 and msgpack.version[1] < 4:
|
||||
self.log.error(
|
||||
"Error: Unsupported msgpack version: %s (<0.4.0), please run `sudo pip install msgpack-python --upgrade`" %
|
||||
"Error: Unsupported msgpack version: %s (<0.4.0), please run `sudo apt-get install python-pip; sudo pip install msgpack-python --upgrade`" %
|
||||
str(msgpack.version)
|
||||
)
|
||||
sys.exit(0)
|
||||
|
|
@ -74,7 +79,7 @@ class ConnectionServer:
|
|||
ip, port = addr
|
||||
|
||||
# Connection flood protection
|
||||
if ip in self.ip_incoming:
|
||||
if ip in self.ip_incoming and ip not in self.whitelist:
|
||||
self.ip_incoming[ip] += 1
|
||||
if self.ip_incoming[ip] > 3: # Allow 3 in 1 minute from same ip
|
||||
self.log.debug("Connection flood detected from %s" % ip)
|
||||
|
|
@ -89,10 +94,15 @@ class ConnectionServer:
|
|||
self.ips[ip] = connection
|
||||
connection.handleIncomingConnection(sock)
|
||||
|
||||
def getConnection(self, ip=None, port=None, peer_id=None, create=True):
|
||||
def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None):
|
||||
if ip.endswith(".onion") and self.tor_manager.start_onions and site: # Site-unique connection for Tor
|
||||
key = ip + site.address
|
||||
else:
|
||||
key = ip
|
||||
|
||||
# Find connection by ip
|
||||
if ip in self.ips:
|
||||
connection = self.ips[ip]
|
||||
if key in self.ips:
|
||||
connection = self.ips[key]
|
||||
if not peer_id or connection.handshake.get("peer_id") == peer_id: # Filter by peer_id
|
||||
if not connection.connected and create:
|
||||
succ = connection.event_connected.get() # Wait for connection
|
||||
|
|
@ -105,6 +115,9 @@ class ConnectionServer:
|
|||
if connection.ip == ip:
|
||||
if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match
|
||||
continue
|
||||
if ip.endswith(".onion") and self.tor_manager.start_onions and connection.site_lock != site.address:
|
||||
# For different site
|
||||
continue
|
||||
if not connection.connected and create:
|
||||
succ = connection.event_connected.get() # Wait for connection
|
||||
if not succ:
|
||||
|
|
@ -116,8 +129,11 @@ class ConnectionServer:
|
|||
if port == 0:
|
||||
raise Exception("This peer is not connectable")
|
||||
try:
|
||||
connection = Connection(self, ip, port)
|
||||
self.ips[ip] = connection
|
||||
if ip.endswith(".onion") and self.tor_manager.start_onions and site: # Lock connection to site
|
||||
connection = Connection(self, ip, port, site_lock=site.address)
|
||||
else:
|
||||
connection = Connection(self, ip, port)
|
||||
self.ips[key] = connection
|
||||
self.connections.append(connection)
|
||||
succ = connection.connect()
|
||||
if not succ:
|
||||
|
|
@ -134,14 +150,22 @@ class ConnectionServer:
|
|||
|
||||
def removeConnection(self, connection):
|
||||
self.log.debug("Removing %s..." % connection)
|
||||
if self.ips.get(connection.ip) == connection: # Delete if same as in registry
|
||||
# Delete if same as in registry
|
||||
if self.ips.get(connection.ip) == connection:
|
||||
del self.ips[connection.ip]
|
||||
# Site locked connection
|
||||
if connection.site_lock and self.ips.get(connection.ip + connection.site_lock) == connection:
|
||||
del self.ips[connection.ip + connection.site_lock]
|
||||
# Cert pinned connection
|
||||
if connection.cert_pin and self.ips.get(connection.ip + "#" + connection.cert_pin) == connection:
|
||||
del self.ips[connection.ip + "#" + connection.cert_pin]
|
||||
|
||||
if connection in self.connections:
|
||||
self.connections.remove(connection)
|
||||
|
||||
def checkConnections(self):
|
||||
while self.running:
|
||||
time.sleep(60) # Sleep 1 min
|
||||
time.sleep(60) # Check every minute
|
||||
self.ip_incoming = {} # Reset connected ips counter
|
||||
self.broken_ssl_peer_ids = {} # Reset broken ssl peerids count
|
||||
for connection in self.connections[:]: # Make a copy
|
||||
|
|
@ -151,7 +175,10 @@ class ConnectionServer:
|
|||
# Delete the unpacker if not needed
|
||||
del connection.unpacker
|
||||
connection.unpacker = None
|
||||
connection.log("Unpacker deleted")
|
||||
|
||||
elif connection.last_cmd == "announce" and idle > 20: # Bootstrapper connection close after 20 sec
|
||||
connection.log("[Cleanup] Tracker connection: %s" % idle)
|
||||
connection.close()
|
||||
|
||||
if idle > 60 * 60:
|
||||
# Wake up after 1h
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue