oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

New bootstraper db format with ipv6 support

Browse source
This commit is contained in:
shortcutme 2019-01-20 16:08:51 +01:00
parent 64e8efbc07
commit 4db723fa6f
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
2 changed files with 78 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
plugins/disabled-Bootstrapper/BootstrapperDb.py
View file

@ -10,7 +10,7 @@ from util import helper
class BootstrapperDb(Db): class BootstrapperDb(Db):
def __init__(self): def __init__(self):
self.version = 6 self.version = 7
self.hash_ids = {} # hash -> id cache self.hash_ids = {} # hash -> id cache
super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, "%s/bootstrapper.db" % config.data_dir) super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, "%s/bootstrapper.db" % config.data_dir)
self.foreign_keys = True self.foreign_keys = True
@ -20,7 +20,7 @@ class BootstrapperDb(Db):
def cleanup(self): def cleanup(self):
while 1: while 1:
time.sleep(4*60) time.sleep(4 * 60)
timeout = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 40)) timeout = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 40))
self.execute("DELETE FROM peer WHERE date_announced < ?", [timeout]) self.execute("DELETE FROM peer WHERE date_announced < ?", [timeout])
@ -47,14 +47,15 @@ class BootstrapperDb(Db):
# Create new tables # Create new tables
self.execute(""" self.execute("""
CREATE TABLE peer ( CREATE TABLE peer (
peer_id INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE, peer_id INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE,
type TEXT,
address TEXT,
port INTEGER NOT NULL, port INTEGER NOT NULL,
ip4 TEXT,
onion TEXT UNIQUE,
date_added DATETIME DEFAULT (CURRENT_TIMESTAMP), date_added DATETIME DEFAULT (CURRENT_TIMESTAMP),
date_announced DATETIME DEFAULT (CURRENT_TIMESTAMP) date_announced DATETIME DEFAULT (CURRENT_TIMESTAMP)
); );
""") """)
self.execute("CREATE UNIQUE INDEX peer_key ON peer (address, port);")
self.execute(""" self.execute("""
CREATE TABLE peer_to_hash ( CREATE TABLE peer_to_hash (
@ -82,19 +83,13 @@ class BootstrapperDb(Db):
self.hash_ids[hash] = self.cur.cursor.lastrowid self.hash_ids[hash] = self.cur.cursor.lastrowid
return self.hash_ids[hash] return self.hash_ids[hash]
def peerAnnounce(self, ip4=None, onion=None, port=None, hashes=[], onion_signed=False, delete_missing_hashes=False): def peerAnnounce(self, ip_type, address, port=None, hashes=[], onion_signed=False, delete_missing_hashes=False):
hashes_ids_announced = [] hashes_ids_announced = []
for hash in hashes: for hash in hashes:
hashes_ids_announced.append(self.getHashId(hash)) hashes_ids_announced.append(self.getHashId(hash))
if not ip4 and not onion:
return 0
# Check user # Check user
if onion: res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"address": address, "port": port})
res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"onion": onion})
else:
res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"ip4": ip4, "port": port})
user_row = res.fetchone() user_row = res.fetchone()
now = time.strftime("%Y-%m-%d %H:%M:%S") now = time.strftime("%Y-%m-%d %H:%M:%S")
@ -102,10 +97,10 @@ class BootstrapperDb(Db):
peer_id = user_row["peer_id"] peer_id = user_row["peer_id"]
self.execute("UPDATE peer SET date_announced = ? WHERE peer_id = ?", (now, peer_id)) self.execute("UPDATE peer SET date_announced = ? WHERE peer_id = ?", (now, peer_id))
else: else:
self.log.debug("New peer: %s %s signed: %s" % (ip4, onion, onion_signed)) self.log.debug("New peer: %s signed: %s" % (address, onion_signed))
if onion and not onion_signed: if ip_type == "onion" and not onion_signed:
return len(hashes) return len(hashes)
self.execute("INSERT INTO peer ?", {"ip4": ip4, "onion": onion, "port": port, "date_announced": now}) self.execute("INSERT INTO peer ?", {"type": ip_type, "address": address, "port": port, "date_announced": now})
peer_id = self.cur.cursor.lastrowid peer_id = self.cur.cursor.lastrowid
# Check user's hashes # Check user's hashes
@ -114,7 +109,7 @@ class BootstrapperDb(Db):
if hash_ids_db != hashes_ids_announced: if hash_ids_db != hashes_ids_announced:
hash_ids_added = set(hashes_ids_announced) - set(hash_ids_db) hash_ids_added = set(hashes_ids_announced) - set(hash_ids_db)
hash_ids_removed = set(hash_ids_db) - set(hashes_ids_announced) hash_ids_removed = set(hash_ids_db) - set(hashes_ids_announced)
if not onion or onion_signed: if ip_type != "onion" or onion_signed:
for hash_id in hash_ids_added: for hash_id in hash_ids_added:
self.execute("INSERT INTO peer_to_hash ?", {"peer_id": peer_id, "hash_id": hash_id}) self.execute("INSERT INTO peer_to_hash ?", {"peer_id": peer_id, "hash_id": hash_id})
if hash_ids_removed and delete_missing_hashes: if hash_ids_removed and delete_missing_hashes:
@ -124,10 +119,10 @@ class BootstrapperDb(Db):
else: else:
return 0 return 0
def peerList(self, hash, ip4=None, onions=[], port=None, limit=30, need_types=["ip4", "onion"], order=True): def peerList(self, hash, address=None, onions=[], port=None, limit=30, need_types=["ipv4", "onion"], order=True):
hash_peers = {"ip4": [], "onion": []} back = {"ipv4": [], "ipv6": [], "onion": []}
if limit == 0: if limit == 0:
return hash_peers return back
hashid = self.getHashId(hash) hashid = self.getHashId(hash)
if order: if order:
@ -137,27 +132,25 @@ class BootstrapperDb(Db):
where_sql = "hash_id = :hashid" where_sql = "hash_id = :hashid"
if onions: if onions:
onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions if type(onion) is str] onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions if type(onion) is str]
where_sql += " AND (onion NOT IN (%s) OR onion IS NULL)" % ",".join(onions_escaped) where_sql += " AND address NOT IN (%s)" % ",".join(onions_escaped)
elif ip4: elif address:
where_sql += " AND (NOT (ip4 = :ip4 AND port = :port) OR ip4 IS NULL)" where_sql += " AND NOT (address = :address AND port = :port)"
query = """ query = """
SELECT ip4, port, onion SELECT type, address, port
FROM peer_to_hash FROM peer_to_hash
LEFT JOIN peer USING (peer_id) LEFT JOIN peer USING (peer_id)
WHERE %s WHERE %s
%s %s
LIMIT :limit LIMIT :limit
""" % (where_sql, order_sql) """ % (where_sql, order_sql)
res = self.execute(query, {"hashid": hashid, "ip4": ip4, "onions": onions, "port": port, "limit": limit}) res = self.execute(query, {"hashid": hashid, "address": address, "port": port, "limit": limit})
for row in res: for row in res:
if row["ip4"] and "ip4" in need_types: if row["type"] in need_types:
hash_peers["ip4"].append( if row["type"] == "onion":
helper.packAddress(row["ip4"], row["port"]) packed = helper.packOnionAddress(row["address"], row["port"])
) else:
if row["onion"] and "onion" in need_types: packed = helper.packAddress(str(row["address"]), row["port"])
hash_peers["onion"].append( back[row["type"]].append(packed)
helper.packOnionAddress(row["onion"], row["port"]) return back
)
return hash_peers

83
plugins/disabled-Bootstrapper/BootstrapperPlugin.py
View file

@ -1,8 +1,11 @@
import time import time
from util import helper
from Plugin import PluginManager from Plugin import PluginManager
from BootstrapperDb import BootstrapperDb from BootstrapperDb import BootstrapperDb
from Crypt import CryptRsa from Crypt import CryptRsa
from Config import config
if "db" not in locals().keys(): # Share during reloads if "db" not in locals().keys(): # Share during reloads
db = BootstrapperDb() db = BootstrapperDb()
@ -10,39 +13,50 @@ if "db" not in locals().keys(): # Share during reloads
@PluginManager.registerTo("FileRequest") @PluginManager.registerTo("FileRequest")
class FileRequestPlugin(object): class FileRequestPlugin(object):
def checkOnionSigns(self, onions, onion_signs, onion_sign_this):
if not onion_signs or len(onion_signs) != len(set(onions)):
return False
if time.time() - float(onion_sign_this) > 3 * 60:
return False # Signed out of allowed 3 minutes
onions_signed = []
# Check onion signs
for onion_publickey, onion_sign in onion_signs.items():
if CryptRsa.verify(onion_sign_this, onion_publickey, onion_sign):
onions_signed.append(CryptRsa.publickeyToOnion(onion_publickey))
else:
break
# Check if the same onion addresses signed as the announced onces
if sorted(onions_signed) == sorted(set(onions)):
return True
else:
return False
def actionAnnounce(self, params): def actionAnnounce(self, params):
time_started = time.time() time_started = time.time()
s = time.time() s = time.time()
# Backward compatibility
if "ip4" in params["add"]:
params["add"].append("ipv4")
if "ip4" in params["need_types"]:
params["need_types"].append("ipv4")
hashes = params["hashes"] hashes = params["hashes"]
if "onion_signs" in params and len(params["onion_signs"]) == len(set(params["onions"])): all_onions_signed = self.checkOnionSigns(params.get("onions", []), params.get("onion_signs"), params.get("onion_sign_this"))
# Check if all sign is correct
if time.time() - float(params["onion_sign_this"]) < 3 * 60: # Peer has 3 minute to sign the message
onions_signed = []
# Check onion signs
for onion_publickey, onion_sign in params["onion_signs"].items():
if CryptRsa.verify(params["onion_sign_this"], onion_publickey, onion_sign):
onions_signed.append(CryptRsa.publickeyToOnion(onion_publickey))
else:
break
# Check if the same onion addresses signed as the announced onces
if sorted(onions_signed) == sorted(set(params["onions"])):
all_onions_signed = True
else:
all_onions_signed = False
else:
# Onion sign this out of 3 minute
all_onions_signed = False
else:
# Incorrect signs number
all_onions_signed = False
time_onion_check = time.time() - s time_onion_check = time.time() - s
if "ip4" in params["add"] and self.connection.ip != "127.0.0.1" and not self.connection.ip.endswith(".onion"): ip_type = helper.getIpType(self.connection.ip)
ip4 = self.connection.ip
if ip_type == "onion" or self.connection.ip in config.ip_local:
is_port_open = False
elif ip_type in params["add"]:
is_port_open = True
else: else:
ip4 = None is_port_open = False
s = time.time() s = time.time()
# Separatley add onions to sites or at once if no onions present # Separatley add onions to sites or at once if no onions present
@ -58,7 +72,8 @@ class FileRequestPlugin(object):
db.execute("BEGIN") db.execute("BEGIN")
for onion, onion_hashes in onion_to_hash.iteritems(): for onion, onion_hashes in onion_to_hash.iteritems():
hashes_changed += db.peerAnnounce( hashes_changed += db.peerAnnounce(
onion=onion, ip_type="onion",
address=onion,
port=params["port"], port=params["port"],
hashes=onion_hashes, hashes=onion_hashes,
onion_signed=all_onions_signed onion_signed=all_onions_signed
@ -67,15 +82,16 @@ class FileRequestPlugin(object):
time_db_onion = time.time() - s time_db_onion = time.time() - s
s = time.time() s = time.time()
# Announce all sites if ip4 defined
if ip4: if is_port_open:
hashes_changed += db.peerAnnounce( hashes_changed += db.peerAnnounce(
ip4=ip4, ip_type=ip_type,
address=self.connection.ip,
port=params["port"], port=params["port"],
hashes=hashes, hashes=hashes,
delete_missing_hashes=params.get("delete") delete_missing_hashes=params.get("delete")
) )
time_db_ip4 = time.time() - s time_db_ip = time.time() - s
s = time.time() s = time.time()
# Query sites # Query sites
@ -97,16 +113,19 @@ class FileRequestPlugin(object):
hash_peers = db.peerList( hash_peers = db.peerList(
hash, hash,
ip4=self.connection.ip, onions=onion_to_hash.keys(), port=params["port"], address=self.connection.ip, onions=onion_to_hash.keys(), port=params["port"],
limit=min(limit, params["need_num"]), need_types=params["need_types"], order=order limit=min(limit, params["need_num"]), need_types=params["need_types"], order=order
) )
if "ip4" in params["need_types"]: # Backward compatibility
hash_peers["ip4"] = hash_peers["ipv4"]
del(hash_peers["ipv4"])
peers.append(hash_peers) peers.append(hash_peers)
time_peerlist = time.time() - s time_peerlist = time.time() - s
back["peers"] = peers back["peers"] = peers
self.connection.log( self.connection.log(
"Announce %s sites (onions: %s, onion_check: %.3fs, db_onion: %.3fs, db_ip4: %.3fs, peerlist: %.3fs, limit: %s)" % "Announce %s sites (onions: %s, onion_check: %.3fs, db_onion: %.3fs, db_ip: %.3fs, peerlist: %.3fs, limit: %s)" %
(len(hashes), len(onion_to_hash), time_onion_check, time_db_onion, time_db_ip4, time_peerlist, limit) (len(hashes), len(onion_to_hash), time_onion_check, time_db_onion, time_db_ip, time_peerlist, limit)
) )
self.response(back) self.response(back)