oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #276 from caryoscelus/git-and-newdir

Browse source 
New directory structure, versioning information and "build" types

- data is separated into config-like files, private files (with keys/user data) and normal site data
- use sensible platform-based defaults for data directories (`~/.local` / `~/AppData` etc)
- support various 'build' types (in particular to differentiate between "portable" and normal setup)
- get versioning info from git
- various code improvements (use `pathlib` instead of string formatting)
- separate Actions from main.py
This commit is contained in:
caryoscelus 2024-05-10 11:48:06 +00:00 committed by GitHub
commit 597bd57bd2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 1018 additions and 692 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -45,3 +45,6 @@ plugins/Multiuser
plugins/NoNewSites
plugins/StemPort
plugins/UiPassword
# Build files
src/Build.py

5
CHANGELOG.md
View file

@ -1,7 +1,7 @@
### zeronet-conservancy 0.7.10+
- disable site-plugins installed for security reasons (@caryoscelus)
- fix downloading geoip db (@caryoscelus)
- python <3.6 is officially unsupported
- python <3.8 is officially unsupported
- SafeRe improvements by @geekless
- remove and don't update muted files (@caryoscelus)
- option to disable port checking (@caryoscelus)
@ -11,6 +11,9 @@
- better fix of local sites leak (@caryoscelus)
- ipython-based repl via --repl for debug/interactive development (@caryoscelus)
- optional blocking of compromised id certificates for spam protection (@caryoscelus)
- changes in directory structure (split data and config, use user directories by default)
- use version information from git if available
- different build types (portable vs package)
- various improvements
### zeronet-conservancy 0.7.10 (2023-07-26) (18d35d3bed4f0683e99)

55
build.py Executable file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env python3
## Copyright (c) 2024 caryoscelus
##
## zeronet-conservancy is free software: you can redistribute it and/or modify it under the
## terms of the GNU General Public License as published by the Free Software
## Foundation, either version 3 of the License, or (at your option) any later version.
##
## zeronet-conservancy is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
## FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
## details.
##
## You should have received a copy of the GNU General Public License along with
## zeronet-conservancy. If not, see <https://www.gnu.org/licenses/>.
##
"""Simple build/bundle script
"""
import argparse
def write_to(args, target):
branch = args.branch
commit = args.commit
if branch is None or commit is None:
from src.util import Git
branch = branch or Git.branch() or 'unknown'
commit = commit or Git.commit() or 'unknown'
target.write('\n'.join([
f"build_type = {args.type!r}",
f"branch = {branch!r}",
f"commit = {commit!r}",
f"version = {args.version!r}",
f"platform = {args.platform!r}",
]))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--type', default='source')
parser.add_argument('--version')
parser.add_argument('--branch')
parser.add_argument('--commit')
parser.add_argument('--platform', default='source')
parser.add_argument('--stdout', action=argparse.BooleanOptionalAction, default=False)
args = parser.parse_args()
if args.stdout:
import sys
target = sys.stdout
else:
target = open('src/Build.py', 'w')
write_to(args, target)
if __name__ == '__main__':
main()

2
plugins/AnnounceShare/AnnounceSharePlugin.py
View file

@ -14,7 +14,7 @@ from util import helper
class TrackerStorage(object):
def __init__(self):
self.log = logging.getLogger("TrackerStorage")
self.file_path = "%s/trackers.json" % config.data_dir
self.file_path = config.start_dir / 'trackers.json'
self.load()
self.time_discover = 0.0
atexit.register(self.save)

2
plugins/AnnounceShare/Test/TestAnnounceShare.py
View file

@ -9,7 +9,7 @@ from Config import config
@pytest.mark.usefixtures("resetTempSettings")
class TestAnnounceShare:
def testAnnounceList(self, file_server):
open("%s/trackers.json" % config.data_dir, "w").write("{}")
(config.start_dir / 'trackers.json').open('w').write('{}')
tracker_storage = AnnounceSharePlugin.tracker_storage
tracker_storage.load()
peer = Peer(file_server.ip, 1544, connection_server=file_server)

2
plugins/Chart/ChartDb.py
View file

@ -6,7 +6,7 @@ import time
class ChartDb(Db):
def __init__(self):
self.version = 2
super(ChartDb, self).__init__(self.getSchema(), "%s/chart.db" % config.data_dir)
super(ChartDb, self).__init__(self.getSchema(), config.start_dir / 'chart.db')
self.foreign_keys = True
self.checkTables()
self.sites = self.loadSites()

10
plugins/ContentFilter/ContentFilterStorage.py
View file

@ -14,7 +14,7 @@ from util import helper
class ContentFilterStorage(object):
def __init__(self, site_manager):
self.log = logging.getLogger("ContentFilterStorage")
self.file_path = "%s/filters.json" % config.data_dir
self.file_path = config.config_dir / 'filters.json'
self.site_manager = site_manager
self.file_content = self.load()
@ -36,12 +36,12 @@ class ContentFilterStorage(object):
def load(self):
# Rename previously used mutes.json -> filters.json
if os.path.isfile("%s/mutes.json" % config.data_dir):
if (config.config_dir / 'mutes.json').is_file():
self.log.info("Renaming mutes.json to filters.json...")
os.rename("%s/mutes.json" % config.data_dir, self.file_path)
if os.path.isfile(self.file_path):
os.rename(config.config_dir / 'mutes.json', self.file_path)
if self.file_path.is_file():
try:
return json.load(open(self.file_path))
return json.load(self.file_path.open())
except Exception as err:
self.log.error("Error loading filters.json: %s" % err)
return None

2
plugins/FilePack/FilePackPlugin.py
View file

@ -44,7 +44,7 @@ class UiRequestPlugin(object):
if ".zip/" in path or ".tar.gz/" in path:
file_obj = None
path_parts = self.parsePath(path)
file_path = "%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"])
file_path = config.data_dir / path_parts["address"] / path_parts["inner_path"]
match = re.match(r"^(.*\.(?:tar.gz|zip))/(.*)", file_path)
archive_path, path_within = match.groups()
if archive_path not in archive_cache:

2
plugins/Sidebar/SidebarPlugin.py
View file

@ -686,7 +686,7 @@ class UiWebsocketPlugin(object):
if sys.platform == "linux":
sys_db_paths += ['/usr/share/GeoIP/' + db_name]
data_dir_db_path = os.path.join(config.data_dir, db_name)
data_dir_db_path = config.start_dir / db_name
db_paths = sys_db_paths + [data_dir_db_path]

2
plugins/disabled-Bootstrapper/BootstrapperDb.py
View file

@ -12,7 +12,7 @@ class BootstrapperDb(Db.Db):
def __init__(self):
self.version = 7
self.hash_ids = {} # hash -> id cache
super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, "%s/bootstrapper.db" % config.data_dir)
super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, config.start_dir / 'bootstrapper.db')
self.foreign_keys = True
self.checkTables()
self.updateHashCache()

2
plugins/disabled-Multiuser/MultiuserPlugin.py
View file

@ -16,7 +16,7 @@ def importPluginnedClasses():
from User import UserManager
try:
local_master_addresses = set(json.load(open("%s/users.json" % config.data_dir)).keys()) # Users in users.json
local_master_addresses = set(json.load((config.private_dir / 'users.json').open()).keys()) # Users in users.json
except Exception as err:
local_master_addresses = set()

5
plugins/disabled-Multiuser/Test/TestMultiuser.py
View file

@ -8,7 +8,8 @@ from User import UserManager
class TestMultiuser:
def testMemorySave(self, user):
# It should not write users to disk
users_before = open("%s/users.json" % config.data_dir).read()
users_json = config.private_dir / 'users.json'
users_before = users_json.open().read()
user = UserManager.user_manager.create()
user.save()
assert open("%s/users.json" % config.data_dir).read() == users_before
assert users_json.open().read() == users_before

521
src/Actions.py Normal file
View file

@ -0,0 +1,521 @@
import logging
import sys
import gevent
from Config import config
from Plugin import PluginManager
@PluginManager.acceptPlugins
class Actions:
def call(self, function_name, kwargs):
logging.info(f'zeronet-conservancy {config.version_full} on Python {sys.version} Gevent {gevent.__version__}')
func = getattr(self, function_name, None)
back = func(**kwargs)
if back:
print(back)
def ipythonThread(self):
import IPython
IPython.embed()
self.gevent_quit.set()
# Default action: Start serving UiServer and FileServer
def main(self):
import main
from File import FileServer
from Ui import UiServer
logging.info("Creating FileServer....")
main.file_server = FileServer()
logging.info("Creating UiServer....")
main.ui_server = UiServer()
main.file_server.ui_server = main.ui_server
# for startup_error in startup_errors:
# logging.error("Startup error: %s" % startup_error)
logging.info("Removing old SSL certs...")
from Crypt import CryptConnection
CryptConnection.manager.removeCerts()
logging.info("Starting servers....")
import threading
self.gevent_quit = threading.Event()
launched_greenlets = [gevent.spawn(main.ui_server.start), gevent.spawn(main.file_server.start), gevent.spawn(main.ui_server.startSiteServer)]
# if --repl, start ipython thread
# FIXME: Unfortunately this leads to exceptions on exit so use with care
if config.repl:
threading.Thread(target=self.ipythonThread).start()
stopped = 0
# Process all greenlets in main thread
while not self.gevent_quit.is_set() and stopped < len(launched_greenlets):
stopped += len(gevent.joinall(launched_greenlets, timeout=1))
# Exited due to repl, so must kill greenlets
if stopped < len(launched_greenlets):
gevent.killall(launched_greenlets, exception=KeyboardInterrupt)
logging.info("All server stopped")
# Site commands
def siteCreate(self, use_master_seed=True):
logging.info("Generating new privatekey (use_master_seed: %s)..." % config.use_master_seed)
from Crypt import CryptBitcoin
if use_master_seed:
from User import UserManager
user = UserManager.user_manager.get()
if not user:
user = UserManager.user_manager.create()
address, address_index, site_data = user.getNewSiteData()
privatekey = site_data["privatekey"]
logging.info("Generated using master seed from users.json, site index: %s" % address_index)
else:
privatekey = CryptBitcoin.newPrivatekey()
address = CryptBitcoin.privatekeyToAddress(privatekey)
logging.info("----------------------------------------------------------------------")
logging.info("Site private key: %s" % privatekey)
logging.info(" !!! ^ Save it now, required to modify the site ^ !!!")
logging.info("Site address: %s" % address)
logging.info("----------------------------------------------------------------------")
while True and not config.batch and not use_master_seed:
if input("? Have you secured your private key? (yes, no) > ").lower() == "yes":
break
else:
logging.info("Please, secure it now, you going to need it to modify your site!")
logging.info("Creating directory structure...")
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
(config.data_dir / address).mkdir()
(config.data_dir / address / 'index.html').open('w').write(f"Hello {address}!")
logging.info("Creating content.json...")
site = Site(address)
extend = {"postmessage_nonce_security": True}
if use_master_seed:
extend["address_index"] = address_index
site.content_manager.sign(privatekey=privatekey, extend=extend)
site.settings["own"] = True
site.saveSettings()
logging.info("Site created!")
def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False, remove_missing_optional=False):
from Site.Site import Site
from Site import SiteManager
from Debug import Debug
SiteManager.site_manager.load()
logging.info("Signing site: %s..." % address)
site = Site(address, allow_create=False)
if not privatekey: # If no privatekey defined
from User import UserManager
user = UserManager.user_manager.get()
if user:
site_data = user.getSiteData(address)
privatekey = site_data.get("privatekey")
else:
privatekey = None
if not privatekey:
# Not found in users.json, ask from console
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
# inner_path can be either relative to site directory or absolute/relative path
if os.path.isabs(inner_path):
full_path = os.path.abspath(inner_path)
else:
full_path = os.path.abspath(config.working_dir + '/' + inner_path)
print(full_path)
if os.path.isfile(full_path):
if address in full_path:
# assuming site address is unique, keep only path after it
inner_path = full_path.split(address+'/')[1]
else:
# oops, file that we found seems to be rogue, so reverting to old behaviour
logging.warning(f'using {inner_path} relative to site directory')
try:
succ = site.content_manager.sign(
inner_path=inner_path, privatekey=privatekey,
update_changed_files=True, remove_missing_optional=remove_missing_optional
)
except Exception as err:
logging.error("Sign error: %s" % Debug.formatException(err))
succ = False
if succ and publish:
self.sitePublish(address, inner_path=inner_path)
def siteVerify(self, address):
import time
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
s = time.time()
logging.info("Verifing site: %s..." % address)
site = Site(address)
bad_files = []
for content_inner_path in site.content_manager.contents:
s = time.time()
logging.info("Verifing %s signature..." % content_inner_path)
error = None
try:
file_correct = site.content_manager.verifyFile(
content_inner_path, site.storage.open(content_inner_path, "rb"), ignore_same=False
)
except Exception as err:
file_correct = False
error = err
if file_correct is True:
logging.info("[OK] %s (Done in %.3fs)" % (content_inner_path, time.time() - s))
else:
logging.error("[ERROR] %s: invalid file: %s!" % (content_inner_path, error))
input("Continue?")
bad_files += content_inner_path
logging.info("Verifying site files...")
bad_files += site.storage.verifyFiles()["bad_files"]
if not bad_files:
logging.info("[OK] All file sha512sum matches! (%.3fs)" % (time.time() - s))
else:
logging.error("[ERROR] Error during verifying site files!")
def dbRebuild(self, address):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
logging.info("Rebuilding site sql cache: %s..." % address)
site = SiteManager.site_manager.get(address)
s = time.time()
try:
site.storage.rebuildDb()
logging.info("Done in %.3fs" % (time.time() - s))
except Exception as err:
logging.error(err)
def dbQuery(self, address, query):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
import json
site = Site(address)
result = []
for row in site.storage.query(query):
result.append(dict(row))
print(json.dumps(result, indent=4))
def siteAnnounce(self, address):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
logging.info("Opening a simple connection server")
from File import FileServer
main.file_server = FileServer("127.0.0.1", 1234)
main.file_server.start()
logging.info("Announcing site %s to tracker..." % address)
site = Site(address)
s = time.time()
site.announce()
print("Response time: %.3fs" % (time.time() - s))
print(site.peers)
def siteDownload(self, address):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
logging.info("Opening a simple connection server")
from File import FileServer
main.file_server = FileServer("127.0.0.1", 1234)
file_server_thread = gevent.spawn(main.file_server.start, check_sites=False)
site = Site(address)
on_completed = gevent.event.AsyncResult()
def onComplete(evt):
evt.set(True)
site.onComplete.once(lambda: onComplete(on_completed))
print("Announcing...")
site.announce()
s = time.time()
print("Downloading...")
site.downloadContent("content.json", check_modifications=True)
print("Downloaded in %.3fs" % (time.time()-s))
def siteNeedFile(self, address, inner_path):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
def checker():
while 1:
s = time.time()
time.sleep(1)
print("Switch time:", time.time() - s)
gevent.spawn(checker)
logging.info("Opening a simple connection server")
from File import FileServer
main.file_server = FileServer("127.0.0.1", 1234)
file_server_thread = gevent.spawn(main.file_server.start, check_sites=False)
site = Site(address)
site.announce()
print(site.needFile(inner_path, update=True))
def siteCmd(self, address, cmd, parameters):
import json
from Site import SiteManager
site = SiteManager.site_manager.get(address)
if not site:
logging.error("Site not found: %s" % address)
return None
ws = self.getWebsocket(site)
ws.send(json.dumps({"cmd": cmd, "params": parameters, "id": 1}))
res_raw = ws.recv()
try:
res = json.loads(res_raw)
except Exception as err:
return {"error": "Invalid result: %s" % err, "res_raw": res_raw}
if "result" in res:
return res["result"]
else:
return res
def importBundle(self, bundle):
import main
main.importBundle(bundle)
def getWebsocket(self, site):
import websocket
ws_address = "ws://%s:%s/Websocket?wrapper_key=%s" % (config.ui_ip, config.ui_port, site.settings["wrapper_key"])
logging.info("Connecting to %s" % ws_address)
ws = websocket.create_connection(ws_address)
return ws
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", recursive=False):
from Site import SiteManager
logging.info("Loading site...")
site = SiteManager.site_manager.get(address)
site.settings["serving"] = True # Serving the site even if its disabled
if not recursive:
inner_paths = [inner_path]
else:
inner_paths = list(site.content_manager.contents.keys())
try:
ws = self.getWebsocket(site)
except Exception as err:
self.sitePublishFallback(site, peer_ip, peer_port, inner_paths, err)
else:
logging.info("Sending siteReload")
self.siteCmd(address, "siteReload", inner_path)
for inner_path in inner_paths:
logging.info(f"Sending sitePublish for {inner_path}")
self.siteCmd(address, "sitePublish", {"inner_path": inner_path, "sign": False})
logging.info("Done.")
ws.close()
def sitePublishFallback(self, site, peer_ip, peer_port, inner_paths, err):
if err is not None:
logging.info(f"Can't connect to local websocket client: {err}")
logging.info("Publish using fallback mechanism. "
"Note that there might be not enough time for peer discovery, "
"but you can specify target peer on command line.")
logging.info("Creating FileServer....")
file_server_thread = gevent.spawn(main.file_server.start, check_sites=False) # Dont check every site integrity
time.sleep(0.001)
# Started fileserver
main.file_server.portCheck()
if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker
logging.info("Gathering peers from tracker")
site.announce() # Gather peers
for inner_path in inner_paths:
published = site.publish(5, inner_path) # Push to peers
if published > 0:
time.sleep(3)
logging.info("Serving files (max 60s)...")
gevent.joinall([file_server_thread], timeout=60)
logging.info("Done.")
else:
logging.info("No peers found, sitePublish command only works if you already have visitors serving your site")
# Crypto commands
def cryptPrivatekeyToAddress(self, privatekey=None):
from Crypt import CryptBitcoin
if not privatekey: # If no privatekey in args then ask it now
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
print(CryptBitcoin.privatekeyToAddress(privatekey))
def cryptSign(self, message, privatekey):
from Crypt import CryptBitcoin
print(CryptBitcoin.sign(message, privatekey))
def cryptVerify(self, message, sign, address):
from Crypt import CryptBitcoin
print(CryptBitcoin.verify(message, address, sign))
def cryptGetPrivatekey(self, master_seed, site_address_index=None):
from Crypt import CryptBitcoin
if len(master_seed) != 64:
logging.error("Error: Invalid master seed length: %s (required: 64)" % len(master_seed))
return False
privatekey = CryptBitcoin.hdPrivatekey(master_seed, site_address_index)
print("Requested private key: %s" % privatekey)
# Peer
def peerPing(self, peer_ip, peer_port=None):
if not peer_port:
peer_port = 15441
logging.info("Opening a simple connection server")
from Connection import ConnectionServer
main.file_server = ConnectionServer("127.0.0.1", 1234)
main.file_server.start(check_connections=False)
from Crypt import CryptConnection
CryptConnection.manager.loadCerts()
from Peer import Peer
logging.info("Pinging 5 times peer: %s:%s..." % (peer_ip, int(peer_port)))
s = time.time()
peer = Peer(peer_ip, peer_port)
peer.connect()
if not peer.connection:
print("Error: Can't connect to peer (connection error: %s)" % peer.connection_error)
return False
if "shared_ciphers" in dir(peer.connection.sock):
print("Shared ciphers:", peer.connection.sock.shared_ciphers())
if "cipher" in dir(peer.connection.sock):
print("Cipher:", peer.connection.sock.cipher()[0])
if "version" in dir(peer.connection.sock):
print("TLS version:", peer.connection.sock.version())
print("Connection time: %.3fs (connection error: %s)" % (time.time() - s, peer.connection_error))
for i in range(5):
ping_delay = peer.ping()
print("Response time: %.3fs" % ping_delay)
time.sleep(1)
peer.remove()
print("Reconnect test...")
peer = Peer(peer_ip, peer_port)
for i in range(5):
ping_delay = peer.ping()
print("Response time: %.3fs" % ping_delay)
time.sleep(1)
def peerGetFile(self, peer_ip, peer_port, site, filename, benchmark=False):
logging.info("Opening a simple connection server")
from Connection import ConnectionServer
main.file_server = ConnectionServer("127.0.0.1", 1234)
main.file_server.start(check_connections=False)
from Crypt import CryptConnection
CryptConnection.manager.loadCerts()
from Peer import Peer
logging.info("Getting %s/%s from peer: %s:%s..." % (site, filename, peer_ip, peer_port))
peer = Peer(peer_ip, peer_port)
s = time.time()
if benchmark:
for i in range(10):
peer.getFile(site, filename),
print("Response time: %.3fs" % (time.time() - s))
input("Check memory")
else:
print(peer.getFile(site, filename).read())
def peerCmd(self, peer_ip, peer_port, cmd, parameters):
logging.info("Opening a simple connection server")
from Connection import ConnectionServer
main.file_server = ConnectionServer()
main.file_server.start(check_connections=False)
from Crypt import CryptConnection
CryptConnection.manager.loadCerts()
from Peer import Peer
peer = Peer(peer_ip, peer_port)
import json
if parameters:
parameters = json.loads(parameters.replace("'", '"'))
else:
parameters = {}
try:
res = peer.request(cmd, parameters)
print(json.dumps(res, indent=2, ensure_ascii=False))
except Exception as err:
print("Unknown response (%s): %s" % (err, res))
def getConfig(self):
import json
print(json.dumps(config.getServerInfo(), indent=2, ensure_ascii=False))
def test(self, test_name, *args, **kwargs):
import types
def funcToName(func_name):
test_name = func_name.replace("test", "")
return test_name[0].lower() + test_name[1:]
test_names = [funcToName(name) for name in dir(self) if name.startswith("test") and name != "test"]
if not test_name:
# No test specificed, list tests
print("\nNo test specified, possible tests:")
for test_name in test_names:
func_name = "test" + test_name[0].upper() + test_name[1:]
func = getattr(self, func_name)
if func.__doc__:
print("- %s: %s" % (test_name, func.__doc__.strip()))
else:
print("- %s" % test_name)
return None
# Run tests
func_name = "test" + test_name[0].upper() + test_name[1:]
if hasattr(self, func_name):
func = getattr(self, func_name)
print("- Running test: %s" % test_name, end="")
s = time.time()
ret = func(*args, **kwargs)
if type(ret) is types.GeneratorType:
for progress in ret:
print(progress, end="")
sys.stdout.flush()
print("\n* Test %s done in %.3fs" % (test_name, time.time() - s))
else:
print("Unknown test: %r (choose from: %s)" % (
test_name, test_names
))

184
src/Config.py
View file

@ -9,23 +9,35 @@ import logging
import logging.handlers
import stat
import time
from pathlib import Path
VERSION = "0.7.10+"
class StartupError(RuntimeError):
pass
class Config:
"""Class responsible for storing and loading config.
Used as singleton `config`
"""
def __init__(self, argv):
try:
from . import Build
except ImportError:
print('cannot find build')
from .util import Git
self.build_type = 'source'
self.branch = Git.branch() or 'unknown'
self.commit = Git.commit() or 'unknown'
self.version = VERSION
self.platform = 'source'
else:
self.build_type = Build.build_type
self.branch = Build.branch
self.commit = Build.commit
self.version = "0.7.10+"
self.version = Build.version or VERSION
self.platform = Build.platform
self.version_full = f'{self.version} ({self.build_type} from {self.branch}-{self.commit})'
self.user_agent = "conservancy"
# for compatibility
@ -43,15 +55,18 @@ class Config:
self.keys_restart_need = set([
"tor", "fileserver_port", "fileserver_ip_type", "threads_fs_read", "threads_fs_write", "threads_crypt", "threads_db"
])
self.start_dir = self.getStartDir()
self.config_file = self.start_dir + "/zeronet.conf"
self.data_dir = self.start_dir + "/data"
self.log_dir = self.start_dir + "/log"
self.config_file = None
self.config_dir = None
self.data_dir = None
self.private_dir = None
self.log_dir = None
self.configurePaths(argv)
self.openssl_lib_file = None
self.openssl_bin_file = None
self.trackers_file = False
self.trackers_file = None
self.createParser()
self.createArguments()
@ -68,7 +83,8 @@ class Config:
def strToBool(self, v):
return v.lower() in ("yes", "true", "t", "1")
def getStartDir(self):
def getStartDirOld(self):
"""Get directory that would have been used by older versions (pre v0.7.11)"""
this_file = os.path.abspath(__file__).replace("\\", "/").rstrip("cd")
if "--start-dir" in self.argv:
@ -89,9 +105,127 @@ class Config:
start_dir = os.path.expanduser("~/ZeroNet")
else:
start_dir = "."
return start_dir
def migrateOld(self, source):
print(f'[bold red]WARNING: found data {source}[/bold red]')
print( ' It used to be default behaviour to store data there,')
print( ' but now we default to place data and config in user home directory.')
print( '')
def configurePaths(self, argv):
if '--config-file' in argv:
self.config_file = argv[argv.index('--config-file') + 1]
old_dir = Path(self.getStartDirOld())
new_dir = Path(self.getStartDir())
no_migrate = '--no-migrate' in argv
silent_migrate = '--portable' in argv or '--migrate' in argv
try:
self.start_dir = self.maybeMigrate(old_dir, new_dir, no_migrate, silent_migrate)
except Exception as ex:
raise ex
self.updatePaths()
def updatePaths(self):
if self.config_file is None:
self.config_file = self.start_dir / 'znc.conf'
if self.config_dir is None:
self.config_dir = self.start_dir
if self.private_dir is None:
self.private_dir = self.start_dir / 'private'
if self.data_dir is None:
self.data_dir = self.start_dir / 'data'
if self.log_dir is None:
self.log_dir = self.start_dir / 'log'
def createPaths(self):
self.start_dir.mkdir(parents=True, exist_ok=True)
self.private_dir.mkdir(parents=True, exist_ok=True)
self.data_dir.mkdir(parents=True, exist_ok=True)
self.log_dir.mkdir(parents=True, exist_ok=True)
def checkDir(self, root):
return (root / 'znc.conf').is_file()
def doMigrate(self, old_dir, new_dir):
raise RuntimeError('Migration not implemented yet')
def askMigrate(self, old_dir, new_dir, silent):
if not sys.stdin.isatty():
raise StartupError('Migration refused: non-interactive shell')
while True:
r = input(f'You have old data in `{old_dir}`. Migrate to new format to `{new_dir}`? [Y/n]')
if r.lower().startswith('n'):
raise StartupError('Migration refused')
if r.lower().startswith('y'):
return self.doMigrate(old_dir, new_dir)
def createNewConfig(self, new_dir):
new_dir.mkdir(parents=True, exist_ok=True)
with (new_dir / 'znc.conf').open('w') as f:
f.write('# zeronet-conervancy config file')
def maybeMigrate(self, old_dir, new_dir, no_migrate, silent_migrate):
if old_dir.exists() and new_dir.exists():
if old_dir == new_dir:
if self.checkDir(new_dir):
return new_dir
elif no_migrate:
return StartError('Migration refused, but new directory should be migrated')
else:
return askMigrate(old_dir, new_dir, silent_migrate)
else:
if self.checkDir(new_dir):
if not no_migrate:
print("There's an old starting directory")
return new_dir
else:
raise StartupError('Bad startup directory')
elif old_dir.exists():
if no_migrate:
self.createNewConfig(new_dir)
return new_dir
else:
return self.askMigrate(old_dir, new_dir, silent_migrate)
elif new_dir.exists():
if self.checkDir(new_dir):
return new_dir
else:
return StartupError('Bad startup directory')
else:
self.createNewConfig(new_dir)
return new_dir
def getStartDir(self):
"""Return directory with config & data"""
if "--start-dir" in self.argv:
return self.argv[self.argv.index("--start-dir") + 1]
here = os.path.dirname(os.path.abspath(__file__).replace("\\", "/")).rstrip('/src')
if '--portable' in self.argv or self.build_type == 'portable':
return here
MACOSX_DIR = '~/Library/Application Support/zeronet-conservancy'
WINDOWS_DIR = '~/AppData/zeronet-conservancy'
LIBREDESKTOP_DIR = '~/.local/share/zeronet-conservancy'
if self.platform == 'source':
if platform.system() == 'Darwin':
path = MACOSX_DIR
elif platform.system() == 'Windows':
path = WINDOWS_DIR
else:
path = LIBREDESKTOP_DIR
elif self.platform == 'macosx':
path = MACOSX_DIR
elif self.platform == 'windows':
path = WINDOWS_DIR
elif self.platform == 'libredesktop':
path = LIBREDESKTOP_DIR
else:
raise RuntimeError(f'UNKNOWN PLATFORM: {self.platform}. Something must have went terribly wrong!')
return os.path.expanduser(path)
# Create command line arguments
def createArguments(self):
try:
@ -109,9 +243,9 @@ class Config:
else:
fix_float_decimals = False
config_file = self.start_dir + "/zeronet.conf"
data_dir = self.start_dir + "/data"
log_dir = self.start_dir + "/log"
config_file = self.config_file
data_dir = self.data_dir
log_dir = self.log_dir
ip_local = ["127.0.0.1", "::1"]
@ -229,9 +363,11 @@ class Config:
self.parser.add_argument('--batch', help="Batch mode (No interactive input for commands)", action='store_true')
self.parser.add_argument('--start-dir', help='Path of working dir for variable content (data, log, .conf)', default=self.start_dir, metavar="path")
self.parser.add_argument('--portable', action=argparse.BooleanOptionalAction)
self.parser.add_argument('--start-dir', help='Path of working dir for variable content (data, log, config)', default=self.start_dir, metavar="path")
self.parser.add_argument('--config-file', help='Path of config file', default=config_file, metavar="path")
self.parser.add_argument('--data-dir', help='Path of data directory', default=data_dir, metavar="path")
self.parser.add_argument('--no-migrate', help='Ignore data directories from old 0net versions', action=argparse.BooleanOptionalAction, default=False)
self.parser.add_argument('--console-log-level', help='Level of logging to console', default="default", choices=["default", "DEBUG", "INFO", "ERROR", "off"])
@ -277,7 +413,7 @@ class Config:
self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip')
self.parser.add_argument('--bootstrap-url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/zeronet-conservancy/zeronet-conservancy/master/bootstrap.url', type=str)
self.parser.add_argument('--disable-bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true')
self.parser.add_argument('--bootstrap', help='Enable downloading bootstrap information from clearnet', action=argparse.BooleanOptionalAction, default=True)
self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*')
self.parser.add_argument('--trackers-file', help='Load torrent trackers dynamically from a file (using Syncronite by default)', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*')
self.parser.add_argument('--trackers-proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable")
@ -328,7 +464,7 @@ class Config:
return self.parser
def loadTrackersFile(self):
if not self.trackers_file:
if self.trackers_file is None:
return None
self.trackers = self.arguments.trackers[:]
@ -338,16 +474,17 @@ class Config:
if trackers_file.startswith("/"): # Absolute
trackers_file_path = trackers_file
elif trackers_file.startswith("{data_dir}"): # Relative to data_dir
trackers_file_path = trackers_file.replace("{data_dir}", self.data_dir)
else: # Relative to zeronet.py
trackers_file_path = self.start_dir + "/" + trackers_file
trackers_file_path = trackers_file.replace('{data_dir}', str(self.data_dir))
else:
# Relative to zeronet.py or something else, unsupported
raise RuntimeError(f'trackers_file should be relative to {{data_dir}} or absolute path (not {trackers_file})')
for line in open(trackers_file_path):
tracker = line.strip()
if "://" in tracker and tracker not in self.trackers:
self.trackers.append(tracker)
except Exception as err:
print("Error loading trackers file: %s" % err)
print(f'Error loading trackers file: {err}')
# Find arguments specified for current action
def getActionArguments(self):
@ -412,6 +549,8 @@ class Config:
self.parseCommandline(argv, silent) # Parse argv
self.setAttributes()
self.updatePaths()
self.createPaths()
if parse_config:
argv = self.parseConfig(argv) # Add arguments from config file
@ -436,7 +575,7 @@ class Config:
for arg in args:
if arg.startswith('--') and '_' in arg:
farg = arg.replace('_', '-')
print(f'WARNING: using deprecated flag in command line: {arg} should be {farg}')
print(f'[bold red]WARNING: using deprecated flag in command line: {arg} should be {farg}[/bold red]')
print('Support for deprecated flags might be removed in the future')
else:
farg = arg
@ -473,9 +612,6 @@ class Config:
def parseConfig(self, argv):
argv = self.fixArgs(argv)
# Find config file path from parameters
if "--config-file" in argv:
self.config_file = argv[argv.index("--config-file") + 1]
# Load config file
if os.path.isfile(self.config_file):
config = configparser.RawConfigParser(allow_no_value=True, strict=False)
@ -518,7 +654,7 @@ class Config:
val = val[:]
if key in ("data_dir", "log_dir", "start_dir", "openssl_bin_file", "openssl_lib_file"):
if val:
val = val.replace("\\", "/")
val = Path(val)
setattr(self, key, val)
def loadPlugins(self):

2
src/Content/ContentDb.py
View file

@ -153,7 +153,7 @@ content_dbs = {}
def getContentDb(path=None):
if not path:
path = "%s/content.db" % config.data_dir
path = config.start_dir / 'content.db'
if path not in content_dbs:
content_dbs[path] = ContentDb(path)
content_dbs[path].init()

18
src/Crypt/CryptConnection.py
View file

@ -24,20 +24,20 @@ class CryptConnectionManager:
self.context_server = None
self.openssl_conf_template = "src/lib/openssl/openssl.cnf"
self.openssl_conf = config.data_dir + "/openssl.cnf"
self.openssl_conf = config.private_dir / "openssl.cnf"
self.openssl_env = {
"OPENSSL_CONF": self.openssl_conf,
"RANDFILE": config.data_dir + "/openssl-rand.tmp"
"RANDFILE": config.private_dir / "openssl-rand.tmp"
}
self.crypt_supported = [] # Supported cryptos
self.cacert_pem = config.data_dir + "/cacert-rsa.pem"
self.cakey_pem = config.data_dir + "/cakey-rsa.pem"
self.cert_pem = config.data_dir + "/cert-rsa.pem"
self.cert_csr = config.data_dir + "/cert-rsa.csr"
self.key_pem = config.data_dir + "/key-rsa.pem"
self.cacert_pem = config.private_dir / "cacert-rsa.pem"
self.cakey_pem = config.private_dir / "cakey-rsa.pem"
self.cert_pem = config.private_dir / "cert-rsa.pem"
self.cert_csr = config.private_dir / "cert-rsa.csr"
self.key_pem = config.private_dir / "key-rsa.pem"
self.log = logging.getLogger("CryptConnectionManager")
self.log.debug("Version: %s" % ssl.OPENSSL_VERSION)
@ -105,8 +105,8 @@ class CryptConnectionManager:
if config.keep_ssl_cert:
return False
for file_name in ["cert-rsa.pem", "key-rsa.pem", "cacert-rsa.pem", "cakey-rsa.pem", "cacert-rsa.srl", "cert-rsa.csr", "openssl-rand.tmp"]:
file_path = "%s/%s" % (config.data_dir, file_name)
if os.path.isfile(file_path):
file_path = config.data_dir / file_name
if file_path.is_file():
os.unlink(file_path)
# Load and create cert files is necessary

2
src/Debug/DebugReloader.py
View file

@ -21,7 +21,7 @@ else:
class DebugReloader:
def __init__(self, paths=None):
if not paths:
paths = ["src", "plugins", config.data_dir + "/__plugins__"]
paths = ["src", "plugins"]
self.log = logging.getLogger("DebugReloader")
self.last_chaged = 0
self.callbacks = []

2
src/Plugin/PluginManager.py
View file

@ -25,7 +25,7 @@ class PluginManager:
self.after_load = [] # Execute functions after loaded plugins
self.function_flags = {} # Flag function for permissions
self.reloading = False
self.config_path = config.data_dir + "/plugins.json"
self.config_path = config.config_dir / 'plugins.json'
self.loadConfig()
self.config.setdefault("builtin", {})

5
src/Site/Site.py
View file

@ -88,9 +88,10 @@ class Site(object):
def loadSettings(self, settings=None):
if not settings:
try:
settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address)
with (config.private_dir / 'sites.json').open() as f:
settings = json.load(f).get(self.address)
except Exception as err:
logging.error(f'Error loading {config.data_dir}/sites.json: {err}')
logging.error(f'Error loading {config.private_dir}/sites.json: {err}')
settings = {}
if settings:
self.settings = settings

6
src/Site/SiteManager.py
View file

@ -38,7 +38,7 @@ class SiteManager(object):
load_s = time.time()
# Load new adresses
try:
json_path = f"{config.data_dir}/sites.json"
json_path = config.private_dir / 'sites.json'
data = json.load(open(json_path))
except Exception as err:
self.log.error(f"Unable to load {json_path}: {err}")
@ -48,7 +48,7 @@ class SiteManager(object):
for address, settings in data.items():
if address not in self.sites:
if os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
if (config.data_dir / address / 'content.json').is_file():
# Root content.json exists, try load site
s = time.time()
try:
@ -121,7 +121,7 @@ class SiteManager(object):
s = time.time()
if data:
helper.atomicWrite("%s/sites.json" % config.data_dir, helper.jsonDumps(data).encode("utf8"))
helper.atomicWrite(config.private_dir / 'sites.json', helper.jsonDumps(data).encode("utf8"))
else:
self.log.debug("Save error: No data")
time_write = time.time() - s

2
src/Site/SiteStorage.py
View file

@ -29,7 +29,7 @@ thread_pool_fs_batch = ThreadPool.ThreadPool(1, name="FS batch")
class SiteStorage(object):
def __init__(self, site, allow_create=True):
self.site = site
self.directory = f'{config.data_dir}/{self.site.address}' # Site data diretory
self.directory = config.data_dir / self.site.address # Site data diretory
self.allowed_dir = os.path.abspath(self.directory) # Only serve file within this dir
self.log = site.log
self.db = None # Db class

2
src/Ui/UiRequest.py
View file

@ -786,7 +786,7 @@ class UiRequest:
address = path_parts["address"]
file_path = "%s/%s/%s" % (config.data_dir, address, path_parts["inner_path"])
file_path = config.data_dir / address / path_parts['inner_path']
if (config.debug or config.merge_media) and file_path.split("/")[-1].startswith("all."):
# If debugging merge *.css to all.css and *.js to all.js

5
src/User/User.py
View file

@ -35,8 +35,9 @@ class User(object):
# Save to data/users.json
@util.Noparallel(queue=True, ignore_class=True)
def save(self):
users_json = config.private_dir / 'users.json'
s = time.time()
users = json.load(open("%s/users.json" % config.data_dir))
users = json.load(open(users_json))
if self.master_address not in users:
users[self.master_address] = {} # Create if not exist
user_data = users[self.master_address]
@ -45,7 +46,7 @@ class User(object):
user_data["sites"] = self.sites
user_data["certs"] = self.certs
user_data["settings"] = self.settings
helper.atomicWrite("%s/users.json" % config.data_dir, helper.jsonDumps(users).encode("utf8"))
helper.atomicWrite(users_json, helper.jsonDumps(users).encode("utf8"))
self.log.debug("Saved in %.3fs" % (time.time() - s))
self.delayed_save_thread = None

6
src/User/UserManager.py
View file

@ -15,7 +15,7 @@ class UserManager(object):
self.users = {}
self.log = logging.getLogger("UserManager")
# Load all user from data/users.json
# Load all user from users.json
def load(self):
if not self.users:
self.users = {}
@ -25,7 +25,7 @@ class UserManager(object):
s = time.time()
# Load new users
try:
json_path = "%s/users.json" % config.data_dir
json_path = config.private_dir / 'users.json'
data = json.load(open(json_path))
except Exception as err:
raise Exception("Unable to load %s: %s" % (json_path, err))
@ -57,7 +57,7 @@ class UserManager(object):
user.saveDelayed()
return user
# List all users from data/users.json
# List all users
# Return: {"usermasteraddr": User}
def list(self):
if self.users == {}: # Not loaded yet

640
src/main.py
View file

@ -4,6 +4,9 @@ import stat
import time
import logging
from util.compat import *
from pathlib import Path
from rich import print
startup_errors = []
def startupError(msg):
@ -12,13 +15,6 @@ def startupError(msg):
# Third party modules
import gevent
if gevent.version_info.major <= 1: # Workaround for random crash when libuv used with threads
try:
if "libev" not in str(gevent.config.loop):
gevent.config.loop = "libev-cext"
except Exception as err:
startupError("Unable to switch gevent loop to libev: %s" % err)
import gevent.monkey
gevent.monkey.patch_all(thread=False, subprocess=False)
@ -33,14 +29,12 @@ def load_config():
# Config parse failed completely, show the help screen and exit
config.parse()
load_config()
def importBundle(bundle):
from zipfile import ZipFile
from Crypt.CryptBitcoin import isValidAddress
import json
sites_json_path = f"{config.data_dir}/sites.json"
sites_json_path = config.private_dir / 'sites.json'
try:
with open(sites_json_path) as f:
sites = json.load(f)
@ -58,31 +52,36 @@ def importBundle(bundle):
map(lambda f: removeprefix(f, prefix).split('/')[0], all_files))))
for d in top_2:
if isValidAddress(d):
logging.info(f'unpack {d} into {config.data_dir}')
print(f'Unpacking {d} into {config.data_dir}')
for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files):
tgt = config.data_dir + '/' + removeprefix(fname, prefix)
logging.info(f'-- {fname} --> {tgt}')
tgt = removeprefix(fname, prefix)
print(f'-- {fname} --> {tgt}')
info = zf.getinfo(fname)
info.filename = tgt
zf.extract(info)
zf.extract(info, path=config.data_dir)
logging.info(f'add site {d}')
sites[d] = {}
else:
logging.info(f'Warning: unknown file in a bundle: {prefix+d}')
print(f'Warning: unknown file in a bundle: {prefix+d}')
with open(sites_json_path, 'w') as f:
json.dump(sites, f)
def init_dirs():
data_dir = config.data_dir
has_data_dir = os.path.isdir(data_dir)
need_bootstrap = not config.disable_bootstrap and (not has_data_dir or not os.path.isfile(f'{data_dir}/sites.json')) and not config.offline
data_dir = Path(config.data_dir)
private_dir = Path(config.private_dir)
need_bootstrap = (config.bootstrap
and not config.offline
and (not data_dir.is_dir() or not (private_dir / 'sites.json').is_file()))
if not has_data_dir:
os.mkdir(data_dir)
try:
os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
except Exception as err:
startupError(f"Can't change permission of {data_dir}: {err}")
# old_users_json = data_dir / 'users.json'
# if old_users_json.is_file():
# print('Migrating existing users.json file to private/')
# old_sites_json = data_dir / 'sites.json'
# if old_sites_json.is_file():
# print('Migrating existing sites.json file to private/')
if not data_dir.is_dir():
data_dir.mkdir(parents=True, exist_ok=True)
if need_bootstrap:
import requests
@ -99,31 +98,38 @@ def init_dirs():
startupError(f"Cannot load boostrap bundle (response status: {response.status_code})")
importBundle(BytesIO(response.content))
sites_json = f"{data_dir}/sites.json"
sites_json = private_dir / 'sites.json'
if not os.path.isfile(sites_json):
with open(sites_json, "w") as f:
f.write("{}")
users_json = f"{data_dir}/users.json"
users_json = private_dir / 'users.json'
if not os.path.isfile(users_json):
with open(users_json, "w") as f:
f.write("{}")
# TODO: GET RID OF TOP-LEVEL CODE!!!
config.initConsoleLogger()
def load_plugins():
from Plugin import PluginManager
PluginManager.plugin_manager.loadPlugins()
config.loadPlugins()
config.parse() # Parse again to add plugin configuration options
try:
def init():
load_config()
config.initConsoleLogger()
try:
init_dirs()
except:
except:
import traceback as tb
print(tb.format_exc())
# at least make sure to print help if we're otherwise so helpless
config.parser.print_help()
# config.parser.print_help()
sys.exit(1)
if config.action == "main":
if config.action == "main":
from util import helper
try:
lock = helper.openLocked(f"{config.data_dir}/lock.pid", "w")
lock = helper.openLocked(config.start_dir / 'lock.pid', "w")
lock.write(f"{os.getpid()}")
except BlockingIOError as err:
startupError(f"Can't open lock file, your 0net client is probably already running, exiting... ({err})")
@ -131,34 +137,28 @@ if config.action == "main":
r = proc.wait()
sys.exit(r)
config.initLogging(console_logging=False)
config.initLogging(console_logging=False)
# Debug dependent configuration
from Debug import DebugHook
from Plugin import PluginManager
# Debug dependent configuration
from Debug import DebugHook
def load_plugins():
PluginManager.plugin_manager.loadPlugins()
config.loadPlugins()
config.parse() # Parse again to add plugin configuration options
load_plugins()
load_plugins()
# Log current config
logging.debug("Config: %s" % config)
# Log current config
logging.debug("Config: %s" % config)
# Modify stack size on special hardwares
if config.stack_size:
# Modify stack size on special hardwares
if config.stack_size:
import threading
threading.stack_size(config.stack_size)
# Use pure-python implementation of msgpack to save CPU
if config.msgpack_purepython:
# Use pure-python implementation of msgpack to save CPU
if config.msgpack_purepython:
os.environ["MSGPACK_PUREPYTHON"] = "True"
# Fix console encoding on Windows
# TODO: check if this is still required
if sys.platform.startswith("win"):
# Fix console encoding on Windows
# TODO: check if this is still required
if sys.platform.startswith("win"):
import subprocess
try:
chcp_res = subprocess.check_output("chcp 65001", shell=True).decode(errors="ignore").strip()
@ -166,8 +166,8 @@ if sys.platform.startswith("win"):
except Exception as err:
logging.error("Error changing console encoding to utf8: %s" % err)
# Socket monkey patch
if config.proxy:
# Socket monkey patch
if config.proxy:
from util import SocksProxy
import urllib.request
logging.info("Patching sockets to socks proxy: %s" % config.proxy)
@ -175,7 +175,7 @@ if config.proxy:
config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost
config.disable_udp = True # UDP not supported currently with proxy
SocksProxy.monkeyPatch(*config.proxy.split(":"))
elif config.tor == "always":
elif config.tor == "always":
from util import SocksProxy
import urllib.request
logging.info("Patching sockets to tor socks proxy: %s" % config.tor_proxy)
@ -183,542 +183,20 @@ elif config.tor == "always":
config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost
SocksProxy.monkeyPatch(*config.tor_proxy_split())
config.disable_udp = True
elif config.bind:
elif config.bind:
bind = config.bind
if ":" not in config.bind:
bind += ":0"
from util import helper
helper.socketBindMonkeyPatch(*bind.split(":"))
# -- Actions --
@PluginManager.acceptPlugins
class Actions:
def call(self, function_name, kwargs):
logging.info(f'zeronet-conservancy {config.version_full} on Python {sys.version} Gevent {gevent.__version__}')
func = getattr(self, function_name, None)
back = func(**kwargs)
if back:
print(back)
def ipythonThread(self):
import IPython
IPython.embed()
self.gevent_quit.set()
# Default action: Start serving UiServer and FileServer
def main(self):
global ui_server, file_server
from File import FileServer
from Ui import UiServer
logging.info("Creating FileServer....")
file_server = FileServer()
logging.info("Creating UiServer....")
ui_server = UiServer()
file_server.ui_server = ui_server
for startup_error in startup_errors:
logging.error("Startup error: %s" % startup_error)
logging.info("Removing old SSL certs...")
from Crypt import CryptConnection
CryptConnection.manager.removeCerts()
logging.info("Starting servers....")
import threading
self.gevent_quit = threading.Event()
launched_greenlets = [gevent.spawn(ui_server.start), gevent.spawn(file_server.start), gevent.spawn(ui_server.startSiteServer)]
# if --repl, start ipython thread
# FIXME: Unfortunately this leads to exceptions on exit so use with care
if config.repl:
threading.Thread(target=self.ipythonThread).start()
stopped = 0
# Process all greenlets in main thread
while not self.gevent_quit.is_set() and stopped < len(launched_greenlets):
stopped += len(gevent.joinall(launched_greenlets, timeout=1))
# Exited due to repl, so must kill greenlets
if stopped < len(launched_greenlets):
gevent.killall(launched_greenlets, exception=KeyboardInterrupt)
logging.info("All server stopped")
# Site commands
def siteCreate(self, use_master_seed=True):
logging.info("Generating new privatekey (use_master_seed: %s)..." % config.use_master_seed)
from Crypt import CryptBitcoin
if use_master_seed:
from User import UserManager
user = UserManager.user_manager.get()
if not user:
user = UserManager.user_manager.create()
address, address_index, site_data = user.getNewSiteData()
privatekey = site_data["privatekey"]
logging.info("Generated using master seed from users.json, site index: %s" % address_index)
else:
privatekey = CryptBitcoin.newPrivatekey()
address = CryptBitcoin.privatekeyToAddress(privatekey)
logging.info("----------------------------------------------------------------------")
logging.info("Site private key: %s" % privatekey)
logging.info(" !!! ^ Save it now, required to modify the site ^ !!!")
logging.info("Site address: %s" % address)
logging.info("----------------------------------------------------------------------")
while True and not config.batch and not use_master_seed:
if input("? Have you secured your private key? (yes, no) > ").lower() == "yes":
break
else:
logging.info("Please, secure it now, you going to need it to modify your site!")
logging.info("Creating directory structure...")
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
os.mkdir("%s/%s" % (config.data_dir, address))
open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address)
logging.info("Creating content.json...")
site = Site(address)
extend = {"postmessage_nonce_security": True}
if use_master_seed:
extend["address_index"] = address_index
site.content_manager.sign(privatekey=privatekey, extend=extend)
site.settings["own"] = True
site.saveSettings()
logging.info("Site created!")
def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False, remove_missing_optional=False):
from Site.Site import Site
from Site import SiteManager
from Debug import Debug
SiteManager.site_manager.load()
logging.info("Signing site: %s..." % address)
site = Site(address, allow_create=False)
if not privatekey: # If no privatekey defined
from User import UserManager
user = UserManager.user_manager.get()
if user:
site_data = user.getSiteData(address)
privatekey = site_data.get("privatekey")
else:
privatekey = None
if not privatekey:
# Not found in users.json, ask from console
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
# inner_path can be either relative to site directory or absolute/relative path
if os.path.isabs(inner_path):
full_path = os.path.abspath(inner_path)
else:
full_path = os.path.abspath(config.working_dir + '/' + inner_path)
print(full_path)
if os.path.isfile(full_path):
if address in full_path:
# assuming site address is unique, keep only path after it
inner_path = full_path.split(address+'/')[1]
else:
# oops, file that we found seems to be rogue, so reverting to old behaviour
logging.warning(f'using {inner_path} relative to site directory')
try:
succ = site.content_manager.sign(
inner_path=inner_path, privatekey=privatekey,
update_changed_files=True, remove_missing_optional=remove_missing_optional
)
except Exception as err:
logging.error("Sign error: %s" % Debug.formatException(err))
succ = False
if succ and publish:
self.sitePublish(address, inner_path=inner_path)
def siteVerify(self, address):
import time
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
s = time.time()
logging.info("Verifing site: %s..." % address)
site = Site(address)
bad_files = []
for content_inner_path in site.content_manager.contents:
s = time.time()
logging.info("Verifing %s signature..." % content_inner_path)
error = None
try:
file_correct = site.content_manager.verifyFile(
content_inner_path, site.storage.open(content_inner_path, "rb"), ignore_same=False
)
except Exception as err:
file_correct = False
error = err
if file_correct is True:
logging.info("[OK] %s (Done in %.3fs)" % (content_inner_path, time.time() - s))
else:
logging.error("[ERROR] %s: invalid file: %s!" % (content_inner_path, error))
input("Continue?")
bad_files += content_inner_path
logging.info("Verifying site files...")
bad_files += site.storage.verifyFiles()["bad_files"]
if not bad_files:
logging.info("[OK] All file sha512sum matches! (%.3fs)" % (time.time() - s))
else:
logging.error("[ERROR] Error during verifying site files!")
def dbRebuild(self, address):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
logging.info("Rebuilding site sql cache: %s..." % address)
site = SiteManager.site_manager.get(address)
s = time.time()
try:
site.storage.rebuildDb()
logging.info("Done in %.3fs" % (time.time() - s))
except Exception as err:
logging.error(err)
def dbQuery(self, address, query):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
import json
site = Site(address)
result = []
for row in site.storage.query(query):
result.append(dict(row))
print(json.dumps(result, indent=4))
def siteAnnounce(self, address):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
logging.info("Opening a simple connection server")
global file_server
from File import FileServer
file_server = FileServer("127.0.0.1", 1234)
file_server.start()
logging.info("Announcing site %s to tracker..." % address)
site = Site(address)
s = time.time()
site.announce()
print("Response time: %.3fs" % (time.time() - s))
print(site.peers)
def siteDownload(self, address):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
logging.info("Opening a simple connection server")
global file_server
from File import FileServer
file_server = FileServer("127.0.0.1", 1234)
file_server_thread = gevent.spawn(file_server.start, check_sites=False)
site = Site(address)
on_completed = gevent.event.AsyncResult()
def onComplete(evt):
evt.set(True)
site.onComplete.once(lambda: onComplete(on_completed))
print("Announcing...")
site.announce()
s = time.time()
print("Downloading...")
site.downloadContent("content.json", check_modifications=True)
print("Downloaded in %.3fs" % (time.time()-s))
def siteNeedFile(self, address, inner_path):
from Site.Site import Site
from Site import SiteManager
SiteManager.site_manager.load()
def checker():
while 1:
s = time.time()
time.sleep(1)
print("Switch time:", time.time() - s)
gevent.spawn(checker)
logging.info("Opening a simple connection server")
global file_server
from File import FileServer
file_server = FileServer("127.0.0.1", 1234)
file_server_thread = gevent.spawn(file_server.start, check_sites=False)
site = Site(address)
site.announce()
print(site.needFile(inner_path, update=True))
def siteCmd(self, address, cmd, parameters):
import json
from Site import SiteManager
site = SiteManager.site_manager.get(address)
if not site:
logging.error("Site not found: %s" % address)
return None
ws = self.getWebsocket(site)
ws.send(json.dumps({"cmd": cmd, "params": parameters, "id": 1}))
res_raw = ws.recv()
try:
res = json.loads(res_raw)
except Exception as err:
return {"error": "Invalid result: %s" % err, "res_raw": res_raw}
if "result" in res:
return res["result"]
else:
return res
def importBundle(self, bundle):
importBundle(bundle)
def getWebsocket(self, site):
import websocket
ws_address = "ws://%s:%s/Websocket?wrapper_key=%s" % (config.ui_ip, config.ui_port, site.settings["wrapper_key"])
logging.info("Connecting to %s" % ws_address)
ws = websocket.create_connection(ws_address)
return ws
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", recursive=False):
from Site import SiteManager
logging.info("Loading site...")
site = SiteManager.site_manager.get(address)
site.settings["serving"] = True # Serving the site even if its disabled
if not recursive:
inner_paths = [inner_path]
else:
inner_paths = list(site.content_manager.contents.keys())
try:
ws = self.getWebsocket(site)
except Exception as err:
self.sitePublishFallback(site, peer_ip, peer_port, inner_paths, err)
else:
logging.info("Sending siteReload")
self.siteCmd(address, "siteReload", inner_path)
for inner_path in inner_paths:
logging.info(f"Sending sitePublish for {inner_path}")
self.siteCmd(address, "sitePublish", {"inner_path": inner_path, "sign": False})
logging.info("Done.")
ws.close()
def sitePublishFallback(self, site, peer_ip, peer_port, inner_paths, err):
if err is not None:
logging.info(f"Can't connect to local websocket client: {err}")
logging.info("Publish using fallback mechanism. "
"Note that there might be not enough time for peer discovery, "
"but you can specify target peer on command line.")
logging.info("Creating FileServer....")
file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity
time.sleep(0.001)
# Started fileserver
file_server.portCheck()
if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker
logging.info("Gathering peers from tracker")
site.announce() # Gather peers
for inner_path in inner_paths:
published = site.publish(5, inner_path) # Push to peers
if published > 0:
time.sleep(3)
logging.info("Serving files (max 60s)...")
gevent.joinall([file_server_thread], timeout=60)
logging.info("Done.")
else:
logging.info("No peers found, sitePublish command only works if you already have visitors serving your site")
# Crypto commands
def cryptPrivatekeyToAddress(self, privatekey=None):
from Crypt import CryptBitcoin
if not privatekey: # If no privatekey in args then ask it now
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
print(CryptBitcoin.privatekeyToAddress(privatekey))
def cryptSign(self, message, privatekey):
from Crypt import CryptBitcoin
print(CryptBitcoin.sign(message, privatekey))
def cryptVerify(self, message, sign, address):
from Crypt import CryptBitcoin
print(CryptBitcoin.verify(message, address, sign))
def cryptGetPrivatekey(self, master_seed, site_address_index=None):
from Crypt import CryptBitcoin
if len(master_seed) != 64:
logging.error("Error: Invalid master seed length: %s (required: 64)" % len(master_seed))
return False
privatekey = CryptBitcoin.hdPrivatekey(master_seed, site_address_index)
print("Requested private key: %s" % privatekey)
# Peer
def peerPing(self, peer_ip, peer_port=None):
if not peer_port:
peer_port = 15441
logging.info("Opening a simple connection server")
global file_server
from Connection import ConnectionServer
file_server = ConnectionServer("127.0.0.1", 1234)
file_server.start(check_connections=False)
from Crypt import CryptConnection
CryptConnection.manager.loadCerts()
from Peer import Peer
logging.info("Pinging 5 times peer: %s:%s..." % (peer_ip, int(peer_port)))
s = time.time()
peer = Peer(peer_ip, peer_port)
peer.connect()
if not peer.connection:
print("Error: Can't connect to peer (connection error: %s)" % peer.connection_error)
return False
if "shared_ciphers" in dir(peer.connection.sock):
print("Shared ciphers:", peer.connection.sock.shared_ciphers())
if "cipher" in dir(peer.connection.sock):
print("Cipher:", peer.connection.sock.cipher()[0])
if "version" in dir(peer.connection.sock):
print("TLS version:", peer.connection.sock.version())
print("Connection time: %.3fs (connection error: %s)" % (time.time() - s, peer.connection_error))
for i in range(5):
ping_delay = peer.ping()
print("Response time: %.3fs" % ping_delay)
time.sleep(1)
peer.remove()
print("Reconnect test...")
peer = Peer(peer_ip, peer_port)
for i in range(5):
ping_delay = peer.ping()
print("Response time: %.3fs" % ping_delay)
time.sleep(1)
def peerGetFile(self, peer_ip, peer_port, site, filename, benchmark=False):
logging.info("Opening a simple connection server")
global file_server
from Connection import ConnectionServer
file_server = ConnectionServer("127.0.0.1", 1234)
file_server.start(check_connections=False)
from Crypt import CryptConnection
CryptConnection.manager.loadCerts()
from Peer import Peer
logging.info("Getting %s/%s from peer: %s:%s..." % (site, filename, peer_ip, peer_port))
peer = Peer(peer_ip, peer_port)
s = time.time()
if benchmark:
for i in range(10):
peer.getFile(site, filename),
print("Response time: %.3fs" % (time.time() - s))
input("Check memory")
else:
print(peer.getFile(site, filename).read())
def peerCmd(self, peer_ip, peer_port, cmd, parameters):
logging.info("Opening a simple connection server")
global file_server
from Connection import ConnectionServer
file_server = ConnectionServer()
file_server.start(check_connections=False)
from Crypt import CryptConnection
CryptConnection.manager.loadCerts()
from Peer import Peer
peer = Peer(peer_ip, peer_port)
import json
if parameters:
parameters = json.loads(parameters.replace("'", '"'))
else:
parameters = {}
try:
res = peer.request(cmd, parameters)
print(json.dumps(res, indent=2, ensure_ascii=False))
except Exception as err:
print("Unknown response (%s): %s" % (err, res))
def getConfig(self):
import json
print(json.dumps(config.getServerInfo(), indent=2, ensure_ascii=False))
def test(self, test_name, *args, **kwargs):
import types
def funcToName(func_name):
test_name = func_name.replace("test", "")
return test_name[0].lower() + test_name[1:]
test_names = [funcToName(name) for name in dir(self) if name.startswith("test") and name != "test"]
if not test_name:
# No test specificed, list tests
print("\nNo test specified, possible tests:")
for test_name in test_names:
func_name = "test" + test_name[0].upper() + test_name[1:]
func = getattr(self, func_name)
if func.__doc__:
print("- %s: %s" % (test_name, func.__doc__.strip()))
else:
print("- %s" % test_name)
return None
# Run tests
func_name = "test" + test_name[0].upper() + test_name[1:]
if hasattr(self, func_name):
func = getattr(self, func_name)
print("- Running test: %s" % test_name, end="")
s = time.time()
ret = func(*args, **kwargs)
if type(ret) is types.GeneratorType:
for progress in ret:
print(progress, end="")
sys.stdout.flush()
print("\n* Test %s done in %.3fs" % (test_name, time.time() - s))
else:
print("Unknown test: %r (choose from: %s)" % (
test_name, test_names
))
init()
from Actions import Actions
actions = Actions()
# Starts here when running zeronet.py
def start():
# Call function
action_kwargs = config.getActionArguments()

2
src/util/Git.py
View file

@ -48,7 +48,7 @@ def _gitted(f):
return lambda *args, **kwargs: None
@_gitted
def commit() -> str:
def commit() -> Optional[str]:
"""Returns git revision, possibly suffixed with -dirty"""
dirty = '-dirty' if _repo.is_dirty() else ''
return f'{_repo.head.commit}{dirty}'

114
src/util/argparseCompat.py Normal file
View file

@ -0,0 +1,114 @@
# This code is taken from CPython Lib/argparse.py and contains BooleanOptionalAction
# for use in py<3.9
# Author: Steven J. Bethard <steven.bethard@gmail.com>.
# New maintainer as of 29 August 2019: Raymond Hettinger <raymond.hettinger@gmail.com>
# Copyright © 2001-2024 Python Software Foundation. All rights reserved.
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
# --------------------------------------------
#
# 1. This LICENSE AGREEMENT is between the Python Software Foundation
# ("PSF"), and the Individual or Organization ("Licensee") accessing and
# otherwise using this software ("Python") in source or binary form and
# its associated documentation.
#
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
# analyze, test, perform and/or display publicly, prepare derivative works,
# distribute, and otherwise use Python alone or in any derivative version,
# provided, however, that PSF's License Agreement and PSF's notice of copyright,
# i.e., "Copyright (c) 2001-2024 Python Software Foundation; All Rights Reserved"
# are retained in Python alone or in any derivative version prepared by Licensee.
#
# 3. In the event Licensee prepares a derivative work that is based on
# or incorporates Python or any part thereof, and wants to make
# the derivative work available to others as provided herein, then
# Licensee hereby agrees to include in any such work a brief summary of
# the changes made to Python.
#
# 4. PSF is making Python available to Licensee on an "AS IS"
# basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
# IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
# INFRINGE ANY THIRD PARTY RIGHTS.
#
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
#
# 6. This License Agreement will automatically terminate upon a material
# breach of its terms and conditions.
#
# 7. Nothing in this License Agreement shall be deemed to create any
# relationship of agency, partnership, or joint venture between PSF and
# Licensee. This License Agreement does not grant permission to use PSF
# trademarks or trade name in a trademark sense to endorse or promote
# products or services of Licensee, or any third party.
#
# 8. By copying, installing or otherwise using Python, Licensee
# agrees to be bound by the terms and conditions of this License
# Agreement.
from argparse import Action
class BooleanOptionalAction(Action):
def __init__(self,
option_strings,
dest,
default=None,
type=_deprecated_default,
choices=_deprecated_default,
required=False,
help=None,
metavar=_deprecated_default,
deprecated=False):
_option_strings = []
for option_string in option_strings:
_option_strings.append(option_string)
if option_string.startswith('--'):
option_string = '--no-' + option_string[2:]
_option_strings.append(option_string)
# We need `_deprecated` special value to ban explicit arguments that
# match default value. Like:
# parser.add_argument('-f', action=BooleanOptionalAction, type=int)
for field_name in ('type', 'choices', 'metavar'):
if locals()[field_name] is not _deprecated_default:
import warnings
warnings._deprecated(
field_name,
"{name!r} is deprecated as of Python 3.12 and will be "
"removed in Python {remove}.",
remove=(3, 14))
if type is _deprecated_default:
type = None
if choices is _deprecated_default:
choices = None
if metavar is _deprecated_default:
metavar = None
super().__init__(
option_strings=_option_strings,
dest=dest,
nargs=0,
default=default,
type=type,
choices=choices,
required=required,
help=help,
metavar=metavar,
deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
if option_string in self.option_strings:
setattr(namespace, self.dest, not option_string.startswith('--no-'))
def format_usage(self):
return ' | '.join(self.option_strings)

6
src/util/compat.py
View file

@ -14,3 +14,9 @@ else:
return s.removeprefix(prefix)
def removesuffix(s, suffix, /):
return s.removesuffix(suffix)
import argparse
if not hasattr(argparse, 'BooleanOptionalAction'):
from .argparseCompat import BooleanOptionalAction
argparse.BooleanOptionalAction = BooleanOptionalAction

22
src/util/helper.py
View file

@ -16,17 +16,17 @@ from Config import config
def atomicWrite(dest, content, mode="wb"):
try:
with open(dest + "-tmpnew", mode) as f:
with open(f'{dest}-tmpnew', mode) as f:
f.write(content)
f.flush()
os.fsync(f.fileno())
if os.path.isfile(dest + "-tmpold"): # Previous incomplete write
os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time())
if os.path.isfile(f'{dest}-tmpold'): # Previous incomplete write
os.rename(f'{dest}-tmpold', f'{dest}-tmpold-{time.time()}')
if os.path.isfile(dest): # Rename old file to -tmpold
os.rename(dest, dest + "-tmpold")
os.rename(dest + "-tmpnew", dest)
if os.path.isfile(dest + "-tmpold"):
os.unlink(dest + "-tmpold") # Remove old file
os.rename(dest, f'{dest}-tmpold')
os.rename(f'{dest}-tmpnew', dest)
if os.path.isfile(f'{dest}-tmpold'):
os.unlink(f'{dest}-tmpold') # Remove old file
return True
except Exception as err:
from Debug import Debug
@ -34,8 +34,8 @@ def atomicWrite(dest, content, mode="wb"):
"File %s write failed: %s, (%s) reverting..." %
(dest, Debug.formatException(err), Debug.formatStack())
)
if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest):
os.rename(dest + "-tmpold", dest)
if os.path.isfile(f'{dest}-tmpold') and not os.path.isfile(dest):
os.rename(f'{dest}-tmpold', dest)
return False
@ -85,7 +85,7 @@ def openLocked(path, mode="wb"):
def getFreeSpace():
free_space = -1
if "statvfs" in dir(os): # Unix
statvfs = os.statvfs(config.data_dir.encode("utf8"))
statvfs = os.statvfs(str(config.data_dir).encode("utf8"))
free_space = statvfs.f_frsize * statvfs.f_bavail
else: # Windows
try:
@ -111,7 +111,7 @@ def shellquote(*args):
if len(args) == 1:
return '"%s"' % args[0].replace('"', "")
else:
return tuple(['"%s"' % arg.replace('"', "") for arg in args])
return tuple(['"%s"' % str(arg).replace('"', "") for arg in args])
def packPeers(peers):

15
zeronet.py
View file

@ -6,11 +6,18 @@ from src.Config import config
# fix further imports from src dir
sys.modules['Config'] = sys.modules['src.Config']
def pyReq():
major = sys.version_info.major
minor = sys.version_info.minor
if major < 3 or (major == 3 and minor < 8):
print("Error: Python 3.8+ is required")
sys.exit(0)
if major == 3 and minor < 11:
print(f"Python 3.11+ is recommended (you're running {sys.version})")
def launch():
'''renamed from main to avoid clashes with main module'''
if sys.version_info.major < 3:
print("Error: Python 3.x is required")
sys.exit(0)
pyReq()
if '--silent' not in sys.argv:
from greet import fancy_greet
@ -27,7 +34,7 @@ def launch():
except Exception as log_err:
print("Failed to log error:", log_err)
traceback.print_exc()
error_log_path = config.log_dir + "/error.log"
error_log_path = config.log_dir / "error.log"
traceback.print_exc(file=open(error_log_path, "w"))
print("---")
print("Please report it: https://github.com/zeronet-conservancy/zeronet-conservancy/issues/new?template=bug-report.md")