rev 196, Configurable data dir, log dir and config file, fix for getFile unhandled exceptions, add http://zeronet.io to readme
This commit is contained in:
parent
1c1d9fb75c
commit
66eca389bf
15 changed files with 69 additions and 53 deletions
|
@ -1,6 +1,6 @@
|
|||
# ZeroNet
|
||||
|
||||
Decentralized websites using Bitcoin crypto and the BitTorrent network
|
||||
Decentralized websites using Bitcoin crypto and the BitTorrent network - http://zeronet.io
|
||||
|
||||
|
||||
## Why?
|
||||
|
|
|
@ -394,7 +394,7 @@ class UiRequestPlugin(object):
|
|||
|
||||
schema = {
|
||||
"db_name": "TestDb",
|
||||
"db_file": "data/benchmark.db",
|
||||
"db_file": "%s/benchmark.db" % config.data_dir,
|
||||
"maps": {
|
||||
".*": {
|
||||
"to_table": {
|
||||
|
@ -415,17 +415,17 @@ class UiRequestPlugin(object):
|
|||
}
|
||||
}
|
||||
|
||||
if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db")
|
||||
if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir)
|
||||
|
||||
with benchmark("Open x 10", 0.13):
|
||||
for i in range(10):
|
||||
db = Db(schema, "data/benchmark.db")
|
||||
db = Db(schema, "%s/benchmark.db" % config.data_dir)
|
||||
db.checkTables()
|
||||
db.close()
|
||||
yield "."
|
||||
|
||||
|
||||
db = Db(schema, "data/benchmark.db")
|
||||
db = Db(schema, "%s/benchmark.db" % config.data_dir)
|
||||
db.checkTables()
|
||||
import json
|
||||
|
||||
|
@ -434,9 +434,9 @@ class UiRequestPlugin(object):
|
|||
data = {"test": []}
|
||||
for i in range(1000): # 1000 line of data
|
||||
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
|
||||
json.dump(data, open("data/test_%s.json" % u, "w"))
|
||||
db.loadJson("data/test_%s.json" % u)
|
||||
os.unlink("data/test_%s.json" % u)
|
||||
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
|
||||
db.loadJson("%s/test_%s.json" % (config.data_dir, u))
|
||||
os.unlink("%s/test_%s.json" % (config.data_dir, u))
|
||||
yield "."
|
||||
|
||||
|
||||
|
@ -448,9 +448,9 @@ class UiRequestPlugin(object):
|
|||
data = {"test": []}
|
||||
for i in range(100): # 1000 line of data
|
||||
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
|
||||
json.dump(data, open("data/test_%s.json" % u, "w"))
|
||||
db.loadJson("data/test_%s.json" % u, cur=cur)
|
||||
os.unlink("data/test_%s.json" % u)
|
||||
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
|
||||
db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
|
||||
os.unlink("%s/test_%s.json" % (config.data_dir, u))
|
||||
if u%10 == 0: yield "."
|
||||
cur.execute("COMMIT")
|
||||
|
||||
|
@ -496,7 +496,7 @@ class UiRequestPlugin(object):
|
|||
|
||||
|
||||
db.close()
|
||||
if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db")
|
||||
if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir)
|
||||
|
||||
gc.collect() # Implicit grabage collection
|
||||
|
||||
|
|
|
@ -11,7 +11,7 @@ log = logging.getLogger("DnschainPlugin")
|
|||
|
||||
@PluginManager.registerTo("SiteManager")
|
||||
class SiteManagerPlugin(object):
|
||||
dns_cache_path = "data/dns_cache.json"
|
||||
dns_cache_path = "%s/dns_cache.json" % config.data_dir
|
||||
dns_cache = None
|
||||
|
||||
# Checks if its a valid address
|
||||
|
|
|
@ -4,7 +4,7 @@ import ConfigParser
|
|||
class Config(object):
|
||||
def __init__(self):
|
||||
self.version = "0.3.0"
|
||||
self.rev = 194
|
||||
self.rev = 196
|
||||
self.parser = self.createArguments()
|
||||
argv = sys.argv[:] # Copy command line arguments
|
||||
argv = self.parseConfig(argv) # Add arguments from config file
|
||||
|
@ -103,12 +103,16 @@ class Config(object):
|
|||
parser.add_argument('--debug', help='Debug mode', action='store_true')
|
||||
parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true')
|
||||
|
||||
parser.add_argument('--config_file', help='Path of config file', default="zeronet.conf", metavar="path")
|
||||
parser.add_argument('--data_dir', help='Path of data directory', default="data", metavar="path")
|
||||
parser.add_argument('--log_dir', help='Path of logging directory', default="log", metavar="path")
|
||||
|
||||
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip')
|
||||
parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
|
||||
parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*')
|
||||
parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name')
|
||||
parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
|
||||
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit')
|
||||
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size')
|
||||
|
||||
parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
|
||||
parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
|
||||
|
@ -151,17 +155,19 @@ class Config(object):
|
|||
action = self.getAction(argv)
|
||||
if len(argv) == 1 or not action: # If no action specificed set the main action
|
||||
argv.append("main")
|
||||
if "zeronet.py" in argv[0]:
|
||||
self.arguments = self.parser.parse_args(argv[1:])
|
||||
else: # Silent errors if not started with zeronet.py
|
||||
self.arguments = self.parser.parse_args(argv[1:])
|
||||
self.arguments = self.parser.parse_args(argv[1:])
|
||||
|
||||
|
||||
# Parse config file
|
||||
def parseConfig(self, argv):
|
||||
if os.path.isfile("zeronet.conf"):
|
||||
# Find config file path from parameters
|
||||
config_file = "zeronet.conf"
|
||||
if "--config_file" in argv:
|
||||
config_file = argv[argv.index("--config_file")+1]
|
||||
# Load config file
|
||||
if os.path.isfile(config_file):
|
||||
config = ConfigParser.ConfigParser(allow_no_value=True)
|
||||
config.read('zeronet.conf')
|
||||
config.read(config_file)
|
||||
for section in config.sections():
|
||||
for key, val in config.items(section):
|
||||
if section != "global": # If not global prefix key with section
|
||||
|
|
|
@ -35,7 +35,7 @@ class DebugReloader:
|
|||
|
||||
|
||||
def changed(self, evt):
|
||||
if not evt.path or "data/" in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec
|
||||
if not evt.path or "%s/" % config.data_dir in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec
|
||||
#logging.debug("Changed: %s" % evt)
|
||||
time.sleep(0.1) # Wait for lock release
|
||||
self.callback()
|
||||
|
|
|
@ -154,7 +154,7 @@ class FileRequest(object):
|
|||
for peer in params["peers"]: # Add sent peers to site
|
||||
address = self.unpackAddress(peer)
|
||||
got_peer_keys.append("%s:%s" % address)
|
||||
if (site.addPeer(*address)): added += 1
|
||||
if site.addPeer(*address): added += 1
|
||||
# Send back peers that is not in the sent list and connectable (not port 0)
|
||||
packed_peers = [peer.packAddress() for peer in site.getConnectablePeers(params["need"], got_peer_keys)]
|
||||
if added:
|
||||
|
|
|
@ -59,7 +59,7 @@ class Site:
|
|||
|
||||
# Load site settings from data/sites.json
|
||||
def loadSettings(self):
|
||||
sites_settings = json.load(open("data/sites.json"))
|
||||
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
|
||||
if self.address in sites_settings:
|
||||
self.settings = sites_settings[self.address]
|
||||
else:
|
||||
|
@ -73,9 +73,9 @@ class Site:
|
|||
|
||||
# Save site settings to data/sites.json
|
||||
def saveSettings(self):
|
||||
sites_settings = json.load(open("data/sites.json"))
|
||||
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
|
||||
sites_settings[self.address] = self.settings
|
||||
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
|
||||
open("%s/sites.json" % config.data_dir, "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
|
||||
return
|
||||
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import json, logging, time, re, os
|
||||
import gevent
|
||||
from Plugin import PluginManager
|
||||
from Config import config
|
||||
|
||||
TRACKERS = [
|
||||
("udp", "open.demonii.com", 1337),
|
||||
|
@ -33,8 +34,8 @@ class SiteManager(object):
|
|||
address_found = []
|
||||
added = 0
|
||||
# Load new adresses
|
||||
for address in json.load(open("data/sites.json")):
|
||||
if address not in self.sites and os.path.isfile("data/%s/content.json" % address):
|
||||
for address in json.load(open("%s/sites.json" % config.data_dir)):
|
||||
if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
|
||||
self.sites[address] = Site(address)
|
||||
added += 1
|
||||
address_found.append(address)
|
||||
|
|
|
@ -2,12 +2,13 @@ import os, re, shutil, json, time, sqlite3
|
|||
import gevent.event
|
||||
from Db import Db
|
||||
from Debug import Debug
|
||||
from Config import config
|
||||
|
||||
|
||||
class SiteStorage:
|
||||
def __init__(self, site, allow_create=True):
|
||||
self.site = site
|
||||
self.directory = "data/%s" % self.site.address # Site data diretory
|
||||
self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory
|
||||
self.log = site.log
|
||||
self.db = None # Db class
|
||||
self.db_checked = False # Checked db tables since startup
|
||||
|
|
|
@ -120,11 +120,11 @@ class TestCase(unittest.TestCase):
|
|||
def testDb(self):
|
||||
print "Importing db..."
|
||||
from Db import Db
|
||||
for db_path in [os.path.abspath("data/test/zeronet.db"), "data/test/zeronet.db"]:
|
||||
for db_path in [os.path.abspath("%s/test/zeronet.db" % config.data_dir), "%s/test/zeronet.db" % config.data_dir]:
|
||||
print "Creating db using %s..." % db_path,
|
||||
schema = {
|
||||
"db_name": "TestDb",
|
||||
"db_file": "data/test/zeronet.db",
|
||||
"db_file": "%s/test/zeronet.db" % config.data_dir,
|
||||
"map": {
|
||||
"data.json": {
|
||||
"to_table": {
|
||||
|
@ -144,14 +144,14 @@ class TestCase(unittest.TestCase):
|
|||
}
|
||||
}
|
||||
|
||||
if os.path.isfile("data/test/zeronet.db"): os.unlink("data/test/zeronet.db")
|
||||
db = Db(schema, "data/test/zeronet.db")
|
||||
if os.path.isfile("%s/test/zeronet.db" % config.data_dir): os.unlink("%s/test/zeronet.db" % config.data_dir)
|
||||
db = Db(schema, "%s/test/zeronet.db" % config.data_dir)
|
||||
db.checkTables()
|
||||
db.close()
|
||||
|
||||
# Cleanup
|
||||
os.unlink("data/test/zeronet.db")
|
||||
os.rmdir("data/test/")
|
||||
os.unlink("%s/test/zeronet.db" % config.data_dir)
|
||||
os.rmdir("%s/test/" % config.data_dir)
|
||||
|
||||
|
||||
def testContentManagerIncludes(self):
|
||||
|
|
|
@ -236,8 +236,8 @@ class UiRequest(object):
|
|||
|
||||
if match: # Looks like a valid path
|
||||
address = match.group("address")
|
||||
file_path = "data/%s/%s" % (address, match.group("inner_path"))
|
||||
allowed_dir = os.path.abspath("data/%s" % address) # Only files within data/sitehash allowed
|
||||
file_path = "%s/%s/%s" % (config.data_dir, address, match.group("inner_path"))
|
||||
allowed_dir = os.path.abspath("%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed
|
||||
data_dir = os.path.abspath("data") # No files from data/ allowed
|
||||
if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir) or allowed_dir == data_dir: # File not in allowed path
|
||||
return self.error403()
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
import logging, json, time
|
||||
from Crypt import CryptBitcoin
|
||||
from Plugin import PluginManager
|
||||
from Config import config
|
||||
|
||||
|
||||
@PluginManager.acceptPlugins
|
||||
class User(object):
|
||||
|
@ -22,13 +24,13 @@ class User(object):
|
|||
|
||||
# Save to data/users.json
|
||||
def save(self):
|
||||
users = json.load(open("data/users.json"))
|
||||
users = json.load(open("%s/users.json" % config.data_dir))
|
||||
if not self.master_address in users: users[self.master_address] = {} # Create if not exits
|
||||
user_data = users[self.master_address]
|
||||
if self.master_seed: user_data["master_seed"] = self.master_seed
|
||||
user_data["sites"] = self.sites
|
||||
user_data["certs"] = self.certs
|
||||
open("data/users.json", "w").write(json.dumps(users, indent=2, sort_keys=True))
|
||||
open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True))
|
||||
self.log.debug("Saved")
|
||||
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import json, logging, os
|
||||
from User import User
|
||||
from Plugin import PluginManager
|
||||
from Config import config
|
||||
|
||||
|
||||
@PluginManager.acceptPlugins
|
||||
|
@ -16,7 +17,7 @@ class UserManager(object):
|
|||
user_found = []
|
||||
added = 0
|
||||
# Load new users
|
||||
for master_address, data in json.load(open("data/users.json")).items():
|
||||
for master_address, data in json.load(open("%s/users.json" % config.data_dir)).items():
|
||||
if master_address not in self.users:
|
||||
user = User(master_address, data=data)
|
||||
self.users[master_address] = user
|
||||
|
|
|
@ -40,7 +40,11 @@ class Worker:
|
|||
self.task = task
|
||||
site = task["site"]
|
||||
task["workers_num"] += 1
|
||||
buff = self.peer.getFile(site.address, task["inner_path"])
|
||||
try:
|
||||
buff = self.peer.getFile(site.address, task["inner_path"])
|
||||
except Exception, err:
|
||||
self.manager.log.debug("%s: getFile error: err" % (self.key, err))
|
||||
buff = None
|
||||
if self.running == False: # Worker no longer needed or got killed
|
||||
self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"]))
|
||||
break
|
||||
|
|
25
src/main.py
25
src/main.py
|
@ -1,22 +1,23 @@
|
|||
import os, sys
|
||||
update_after_shutdown = False # If set True then update and restart zeronet after main loop ended
|
||||
|
||||
# Create necessary files and dirs
|
||||
if not os.path.isdir("log"): os.mkdir("log")
|
||||
if not os.path.isdir("data"): os.mkdir("data")
|
||||
if not os.path.isfile("data/sites.json"): open("data/sites.json", "w").write("{}")
|
||||
if not os.path.isfile("data/users.json"): open("data/users.json", "w").write("{}")
|
||||
|
||||
# Load config
|
||||
from Config import config
|
||||
|
||||
# Create necessary files and dirs
|
||||
if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir)
|
||||
if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir)
|
||||
if not os.path.isfile("%s/sites.json" % config.data_dir): open("%s/sites.json" % config.data_dir, "w").write("{}")
|
||||
if not os.path.isfile("%s/users.json" % config.data_dir): open("%s/users.json" % config.data_dir, "w").write("{}")
|
||||
|
||||
|
||||
# Setup logging
|
||||
import logging
|
||||
if config.action == "main":
|
||||
if os.path.isfile("log/debug.log"): # Simple logrotate
|
||||
if os.path.isfile("log/debug-last.log"): os.unlink("log/debug-last.log")
|
||||
os.rename("log/debug.log", "log/debug-last.log")
|
||||
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="log/debug.log")
|
||||
if os.path.isfile("%s/debug.log" % config.log_dir): # Simple logrotate
|
||||
if os.path.isfile("%s/debug-last.log" % config.log_dir): os.unlink("%s/debug-last.log" % config.log_dir)
|
||||
os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir)
|
||||
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="%s/debug.log" % config.log_dir)
|
||||
else:
|
||||
logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main
|
||||
|
||||
|
@ -99,8 +100,8 @@ class Actions:
|
|||
|
||||
logging.info("Creating directory structure...")
|
||||
from Site import Site
|
||||
os.mkdir("data/%s" % address)
|
||||
open("data/%s/index.html" % address, "w").write("Hello %s!" % address)
|
||||
os.mkdir("%s/%s" % (config.data_dir, address))
|
||||
open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address)
|
||||
|
||||
logging.info("Creating content.json...")
|
||||
site = Site(address)
|
||||
|
|
Loading…
Reference in a new issue