Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol

This commit is contained in:
HelloZeroNet 2016-01-05 00:20:52 +01:00
parent c9578e9037
commit e9d2cdfd37
99 changed files with 9476 additions and 267 deletions

View file

@ -0,0 +1,157 @@
import time
import re
import gevent
from Config import config
from Db import Db
from util import helper
class BootstrapperDb(Db):
def __init__(self):
self.version = 6
self.hash_ids = {} # hash -> id cache
super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, "%s/bootstrapper.db" % config.data_dir)
self.foreign_keys = True
self.checkTables()
self.updateHashCache()
gevent.spawn(self.cleanup)
def cleanup(self):
while 1:
self.execute("DELETE FROM peer WHERE date_announced < DATETIME('now', '-40 minute')")
time.sleep(4*60)
def updateHashCache(self):
res = self.execute("SELECT * FROM hash")
self.hash_ids = {str(row["hash"]): row["hash_id"] for row in res}
self.log.debug("Loaded %s hash_ids" % len(self.hash_ids))
def checkTables(self):
version = int(self.execute("PRAGMA user_version").fetchone()[0])
self.log.debug("Db version: %s, needed: %s" % (version, self.version))
if version < self.version:
self.createTables()
else:
self.execute("VACUUM")
def createTables(self):
# Delete all tables
self.execute("PRAGMA writable_schema = 1")
self.execute("DELETE FROM sqlite_master WHERE type IN ('table', 'index', 'trigger')")
self.execute("PRAGMA writable_schema = 0")
self.execute("VACUUM")
self.execute("PRAGMA INTEGRITY_CHECK")
# Create new tables
self.execute("""
CREATE TABLE peer (
peer_id INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE,
port INTEGER NOT NULL,
ip4 TEXT,
onion TEXT,
date_added DATETIME DEFAULT (CURRENT_TIMESTAMP),
date_announced DATETIME DEFAULT (CURRENT_TIMESTAMP)
);
""")
self.execute("""
CREATE TABLE peer_to_hash (
peer_to_hash_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE NOT NULL,
peer_id INTEGER REFERENCES peer (peer_id) ON DELETE CASCADE,
hash_id INTEGER REFERENCES hash (hash_id)
);
""")
self.execute("CREATE INDEX peer_id ON peer_to_hash (peer_id);")
self.execute("CREATE INDEX hash_id ON peer_to_hash (hash_id);")
self.execute("""
CREATE TABLE hash (
hash_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE NOT NULL,
hash BLOB UNIQUE NOT NULL,
date_added DATETIME DEFAULT (CURRENT_TIMESTAMP)
);
""")
self.execute("PRAGMA user_version = %s" % self.version)
def getHashId(self, hash):
if hash not in self.hash_ids:
self.log.debug("New hash: %s" % repr(hash))
self.execute("INSERT OR IGNORE INTO hash ?", {"hash": buffer(hash)})
self.hash_ids[hash] = self.cur.cursor.lastrowid
return self.hash_ids[hash]
def peerAnnounce(self, ip4=None, onion=None, port=None, hashes=[], onion_signed=False, delete_missing_hashes=False):
hashes_ids_announced = []
for hash in hashes:
hashes_ids_announced.append(self.getHashId(hash))
if not ip4 and not onion:
return 0
# Check user
if onion:
res = self.execute("SELECT * FROM peer WHERE ? LIMIT 1", {"onion": onion})
else:
res = self.execute("SELECT * FROM peer WHERE ? LIMIT 1", {"ip4": ip4, "port": port})
user_row = res.fetchone()
if user_row:
peer_id = user_row["peer_id"]
self.execute("UPDATE peer SET date_announced = DATETIME('now') WHERE ?", {"peer_id": peer_id})
else:
self.log.debug("New peer: %s %s signed: %s" % (ip4, onion, onion_signed))
if onion and not onion_signed:
return len(hashes)
self.execute("INSERT INTO peer ?", {"ip4": ip4, "onion": onion, "port": port})
peer_id = self.cur.cursor.lastrowid
# Check user's hashes
res = self.execute("SELECT * FROM peer_to_hash WHERE ?", {"peer_id": peer_id})
hash_ids_db = [row["hash_id"] for row in res]
if hash_ids_db != hashes_ids_announced:
hash_ids_added = set(hashes_ids_announced) - set(hash_ids_db)
hash_ids_removed = set(hash_ids_db) - set(hashes_ids_announced)
if not onion or onion_signed:
for hash_id in hash_ids_added:
self.execute("INSERT INTO peer_to_hash ?", {"peer_id": peer_id, "hash_id": hash_id})
if hash_ids_removed and delete_missing_hashes:
self.execute("DELETE FROM peer_to_hash WHERE ?", {"peer_id": peer_id, "hash_id": list(hash_ids_removed)})
return len(hash_ids_added) + len(hash_ids_removed)
else:
return 0
def peerList(self, hash, ip4=None, onions=[], port=None, limit=30, need_types=["ip4", "onion"]):
hash_peers = {"ip4": [], "onion": []}
if limit == 0:
return hash_peers
hashid = self.getHashId(hash)
where = "hash_id = :hashid"
if onions:
onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions]
where += " AND (onion NOT IN (%s) OR onion IS NULL)" % ",".join(onions_escaped)
elif ip4:
where += " AND (NOT (ip4 = :ip4 AND port = :port) OR ip4 IS NULL)"
query = """
SELECT ip4, port, onion
FROM peer_to_hash
LEFT JOIN peer USING (peer_id)
WHERE %s
LIMIT :limit
""" % where
res = self.execute(query, {"hashid": hashid, "ip4": ip4, "onions": onions, "port": port, "limit": limit})
for row in res:
if row["ip4"] and "ip4" in need_types:
hash_peers["ip4"].append(
helper.packAddress(row["ip4"], row["port"])
)
if row["onion"] and "onion" in need_types:
hash_peers["onion"].append(
helper.packOnionAddress(row["onion"], row["port"])
)
return hash_peers

View file

@ -0,0 +1,105 @@
import time
from Plugin import PluginManager
from BootstrapperDb import BootstrapperDb
from Crypt import CryptRsa
if "db" not in locals().keys(): # Share durin reloads
db = BootstrapperDb()
@PluginManager.registerTo("FileRequest")
class FileRequestPlugin(object):
def actionAnnounce(self, params):
hashes = params["hashes"]
if "onion_signs" in params and len(params["onion_signs"]) == len(hashes):
# Check if all sign is correct
if time.time() - float(params["onion_sign_this"]) < 3*60: # Peer has 3 minute to sign the message
onions_signed = []
# Check onion signs
for onion_publickey, onion_sign in params["onion_signs"].items():
if CryptRsa.verify(params["onion_sign_this"], onion_publickey, onion_sign):
onions_signed.append(CryptRsa.publickeyToOnion(onion_publickey))
else:
break
# Check if the same onion addresses signed as the announced onces
if sorted(onions_signed) == sorted(params["onions"]):
all_onions_signed = True
else:
all_onions_signed = False
else:
# Onion sign this out of 3 minute
all_onions_signed = False
else:
# Incorrect signs number
all_onions_signed = False
if "ip4" in params["add"] and self.connection.ip != "127.0.0.1" and not self.connection.ip.endswith(".onion"):
ip4 = self.connection.ip
else:
ip4 = None
# Separatley add onions to sites or at once if no onions present
hashes_changed = 0
i = 0
for onion in params.get("onions", []):
hashes_changed += db.peerAnnounce(
onion=onion,
port=params["port"],
hashes=[hashes[i]],
onion_signed=all_onions_signed
)
i += 1
# Announce all sites if ip4 defined
if ip4:
hashes_changed += db.peerAnnounce(
ip4=ip4,
port=params["port"],
hashes=hashes,
delete_missing_hashes=params.get("delete")
)
# Query sites
back = {}
peers = []
if params.get("onions") and not all_onions_signed and hashes_changed:
back["onion_sign_this"] = "%.0f" % time.time() # Send back nonce for signing
for hash in hashes:
hash_peers = db.peerList(
hash,
ip4=self.connection.ip, onions=params.get("onions"), port=params["port"],
limit=min(30, params["need_num"]), need_types=params["need_types"]
)
peers.append(hash_peers)
back["peers"] = peers
self.response(back)
@PluginManager.registerTo("UiRequest")
class UiRequestPlugin(object):
def actionStatsBootstrapper(self):
self.sendHeader()
# Style
yield """
<style>
* { font-family: monospace; white-space: pre }
table td, table th { text-align: right; padding: 0px 10px }
</style>
"""
hash_rows = db.execute("SELECT * FROM hash").fetchall()
for hash_row in hash_rows:
peer_rows = db.execute(
"SELECT * FROM peer LEFT JOIN peer_to_hash USING (peer_id) WHERE hash_id = :hash_id",
{"hash_id": hash_row["hash_id"]}
).fetchall()
yield "<br>%s (added: %s, peers: %s)<br>" % (
str(hash_row["hash"]).encode("hex"), hash_row["date_added"], len(peer_rows)
)
for peer_row in peer_rows:
yield " - {ip4: <30} {onion: <30} added: {date_added}, announced: {date_announced}<br>".format(**dict(peer_row))

View file

@ -0,0 +1,179 @@
import hashlib
import os
import pytest
from Bootstrapper import BootstrapperPlugin
from Bootstrapper.BootstrapperDb import BootstrapperDb
from Peer import Peer
from Crypt import CryptRsa
from util import helper
@pytest.fixture()
def bootstrapper_db(request):
BootstrapperPlugin.db.close()
BootstrapperPlugin.db = BootstrapperDb()
BootstrapperPlugin.db.createTables() # Reset db
BootstrapperPlugin.db.cur.logging = True
def cleanup():
BootstrapperPlugin.db.close()
os.unlink(BootstrapperPlugin.db.db_path)
request.addfinalizer(cleanup)
return BootstrapperPlugin.db
@pytest.mark.usefixtures("resetSettings")
class TestBootstrapper:
def testIp4(self, file_server, bootstrapper_db):
peer = Peer("127.0.0.1", 1544, connection_server=file_server)
hash1 = hashlib.sha256("site1").digest()
hash2 = hashlib.sha256("site2").digest()
hash3 = hashlib.sha256("site3").digest()
# Verify empty result
res = peer.request("announce", {
"hashes": [hash1, hash2],
"port": 15441, "need_types": ["ip4"], "need_num": 10, "add": ["ip4"]
})
assert len(res["peers"][0]["ip4"]) == 0 # Empty result
# Verify added peer on previous request
bootstrapper_db.peerAnnounce(ip4="1.2.3.4", port=15441, hashes=[hash1, hash2], delete_missing_hashes=True)
res = peer.request("announce", {
"hashes": [hash1, hash2],
"port": 15441, "need_types": ["ip4"], "need_num": 10, "add": ["ip4"]
})
assert len(res["peers"][0]["ip4"]) == 1
assert len(res["peers"][1]["ip4"]) == 1
# hash2 deleted from 1.2.3.4
bootstrapper_db.peerAnnounce(ip4="1.2.3.4", port=15441, hashes=[hash1], delete_missing_hashes=True)
res = peer.request("announce", {
"hashes": [hash1, hash2],
"port": 15441, "need_types": ["ip4"], "need_num": 10, "add": ["ip4"]
})
assert len(res["peers"][0]["ip4"]) == 1
assert len(res["peers"][1]["ip4"]) == 0
# Announce 3 hash again
bootstrapper_db.peerAnnounce(ip4="1.2.3.4", port=15441, hashes=[hash1, hash2, hash3], delete_missing_hashes=True)
res = peer.request("announce", {
"hashes": [hash1, hash2, hash3],
"port": 15441, "need_types": ["ip4"], "need_num": 10, "add": ["ip4"]
})
assert len(res["peers"][0]["ip4"]) == 1
assert len(res["peers"][1]["ip4"]) == 1
assert len(res["peers"][2]["ip4"]) == 1
# Single hash announce
res = peer.request("announce", {
"hashes": [hash1], "port": 15441, "need_types": ["ip4"], "need_num": 10, "add": ["ip4"]
})
assert len(res["peers"][0]["ip4"]) == 1
# Test DB cleanup
assert bootstrapper_db.execute("SELECT COUNT(*) AS num FROM peer").fetchone()["num"] == 1 # 127.0.0.1 never get added to db
# Delete peers
bootstrapper_db.execute("DELETE FROM peer WHERE ip4 = '1.2.3.4'")
assert bootstrapper_db.execute("SELECT COUNT(*) AS num FROM peer_to_hash").fetchone()["num"] == 0
assert bootstrapper_db.execute("SELECT COUNT(*) AS num FROM hash").fetchone()["num"] == 3 # 3 sites
assert bootstrapper_db.execute("SELECT COUNT(*) AS num FROM peer").fetchone()["num"] == 0 # 0 peer
def testPassive(self, file_server, bootstrapper_db):
peer = Peer("127.0.0.1", 1544, connection_server=file_server)
hash1 = hashlib.sha256("hash1").digest()
bootstrapper_db.peerAnnounce(ip4=None, port=15441, hashes=[hash1])
res = peer.request("announce", {
"hashes": [hash1], "port": 15441, "need_types": ["ip4"], "need_num": 10, "add": []
})
assert len(res["peers"][0]["ip4"]) == 0 # Empty result
def testAddOnion(self, file_server, site, bootstrapper_db, tor_manager):
onion1 = tor_manager.addOnion()
onion2 = tor_manager.addOnion()
peer = Peer("127.0.0.1", 1544, connection_server=file_server)
hash1 = hashlib.sha256("site1").digest()
hash2 = hashlib.sha256("site2").digest()
bootstrapper_db.peerAnnounce(ip4="1.2.3.4", port=1234, hashes=[hash1, hash2])
res = peer.request("announce", {
"onions": [onion1, onion2],
"hashes": [hash1, hash2], "port": 15441, "need_types": ["ip4", "onion"], "need_num": 10, "add": ["onion"]
})
assert len(res["peers"][0]["ip4"]) == 1
assert "onion_sign_this" in res
# Onion address not added yet
site_peers = bootstrapper_db.peerList(ip4="1.2.3.4", port=1234, hash=hash1)
assert len(site_peers["onion"]) == 0
assert "onion_sign_this" in res
# Sign the nonces
sign1 = CryptRsa.sign(res["onion_sign_this"], tor_manager.getPrivatekey(onion1))
sign2 = CryptRsa.sign(res["onion_sign_this"], tor_manager.getPrivatekey(onion2))
# Bad sign (different address)
res = peer.request("announce", {
"onions": [onion1], "onion_sign_this": res["onion_sign_this"],
"onion_signs": {tor_manager.getPublickey(onion2): sign2},
"hashes": [hash1], "port": 15441, "need_types": ["ip4", "onion"], "need_num": 10, "add": ["onion"]
})
assert "onion_sign_this" in res
site_peers1 = bootstrapper_db.peerList(ip4="1.2.3.4", port=1234, hash=hash1)
assert len(site_peers1["onion"]) == 0 # Not added
# Bad sign (missing one)
res = peer.request("announce", {
"onions": [onion1, onion2], "onion_sign_this": res["onion_sign_this"],
"onion_signs": {tor_manager.getPublickey(onion1): sign1},
"hashes": [hash1, hash2], "port": 15441, "need_types": ["ip4", "onion"], "need_num": 10, "add": ["onion"]
})
assert "onion_sign_this" in res
site_peers1 = bootstrapper_db.peerList(ip4="1.2.3.4", port=1234, hash=hash1)
assert len(site_peers1["onion"]) == 0 # Not added
# Good sign
res = peer.request("announce", {
"onions": [onion1, onion2], "onion_sign_this": res["onion_sign_this"],
"onion_signs": {tor_manager.getPublickey(onion1): sign1, tor_manager.getPublickey(onion2): sign2},
"hashes": [hash1, hash2], "port": 15441, "need_types": ["ip4", "onion"], "need_num": 10, "add": ["onion"]
})
assert "onion_sign_this" not in res
# Onion addresses added
site_peers1 = bootstrapper_db.peerList(ip4="1.2.3.4", port=1234, hash=hash1)
assert len(site_peers1["onion"]) == 1
site_peers2 = bootstrapper_db.peerList(ip4="1.2.3.4", port=1234, hash=hash2)
assert len(site_peers2["onion"]) == 1
assert site_peers1["onion"][0] != site_peers2["onion"][0]
assert helper.unpackOnionAddress(site_peers1["onion"][0])[0] == onion1+".onion"
assert helper.unpackOnionAddress(site_peers2["onion"][0])[0] == onion2+".onion"
tor_manager.delOnion(onion1)
tor_manager.delOnion(onion2)
def testRequestPeers(self, file_server, site, bootstrapper_db, tor_manager):
site.connection_server = file_server
hash = hashlib.sha256(site.address).digest()
# Request peers from tracker
assert len(site.peers) == 0
bootstrapper_db.peerAnnounce(ip4="1.2.3.4", port=1234, hashes=[hash])
site.announceTracker("zero", "127.0.0.1:1544")
assert len(site.peers) == 1
# Test onion address store
bootstrapper_db.peerAnnounce(onion="bka4ht2bzxchy44r", port=1234, hashes=[hash], onion_signed=True)
site.announceTracker("zero", "127.0.0.1:1544")
assert len(site.peers) == 2
assert "bka4ht2bzxchy44r.onion:1234" in site.peers

View file

@ -0,0 +1 @@
from src.Test.conftest import *

View file

@ -0,0 +1,5 @@
[pytest]
python_files = Test*.py
addopts = -rsxX -v --durations=6
markers =
webtest: mark a test as a webtest.

View file

@ -0,0 +1 @@
import BootstrapperPlugin