diff --git a/plugins/Zeroname/SiteManagerPlugin.py b/plugins/Zeroname/SiteManagerPlugin.py new file mode 100644 index 00000000..806d9021 --- /dev/null +++ b/plugins/Zeroname/SiteManagerPlugin.py @@ -0,0 +1,72 @@ +import logging, json, os, re, sys, time +import gevent +from Plugin import PluginManager +from Config import config +from Debug import Debug + +allow_reload = False # No reload supported + +log = logging.getLogger("ZeronamePlugin") + + +@PluginManager.registerTo("SiteManager") +class SiteManagerPlugin(object): + zeroname_address = "1Name2NXVi1RDPDgf5617UoW7xA6YrhM9F" + site_zeroname = None + + # Checks if its a valid address + def isAddress(self, address): + if self.isDomain(address): + return True + else: + return super(SiteManagerPlugin, self).isAddress(address) + + + # Return: True if the address is domain + def isDomain(self, address): + return re.match("(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9]+)$", address) + + + # Resolve domain + # Return: The address or None + def resolveDomain(self, domain): + domain = domain.lower() + if not self.site_zeroname: + self.site_zeroname = self.need(self.zeroname_address) + self.site_zeroname.needFile("data/names.json", priority=10) + db = self.site_zeroname.storage.loadJson("data/names.json") + return db.get(domain) + + + # Return or create site and start download site files + # Return: Site or None if dns resolve failed + def need(self, address, all_file=True): + if self.isDomain(address): # Its looks like a domain + address_resolved = self.resolveDomain(address) + if address_resolved: + address = address_resolved + else: + return None + + return super(SiteManagerPlugin, self).need(address, all_file) + + + # Return: Site object or None if not found + def get(self, address): + if self.sites == None: # Not loaded yet + self.load() + if self.isDomain(address): # Its looks like a domain + address_resolved = self.resolveDomain(address) + if address_resolved: # Domain found + site = self.sites.get(address_resolved) + if site: + site_domain = site.settings.get("domain") + if site_domain != address: + site.settings["domain"] = address + else: # Domain not found + site = self.sites.get(address) + + else: # Access by site address + site = self.sites.get(address) + return site + diff --git a/plugins/Zeroname/UiRequestPlugin.py b/plugins/Zeroname/UiRequestPlugin.py new file mode 100644 index 00000000..65a386f1 --- /dev/null +++ b/plugins/Zeroname/UiRequestPlugin.py @@ -0,0 +1,34 @@ +import re +from Plugin import PluginManager + +@PluginManager.registerTo("UiRequest") +class UiRequestPlugin(object): + def __init__(self, server = None): + from Site import SiteManager + self.site_manager = SiteManager.site_manager + super(UiRequestPlugin, self).__init__(server) + + + # Media request + def actionSiteMedia(self, path): + match = re.match("/media/(?P
[A-Za-z0-9]+\.[A-Za-z0-9\.]+)(?P/.*|$)", path) + if match: # Its a valid domain, resolve first + domain = match.group("address") + address = self.site_manager.resolveDomain(domain) + if address: + path = "/media/"+address+match.group("inner_path") + return super(UiRequestPlugin, self).actionSiteMedia(path) # Get the wrapper frame output + + + # Is mediarequest allowed from that referer + def isMediaRequestAllowed(self, site_address, referer): + referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address + referer_site_address = re.match("/(?P
[A-Za-z0-9\.]+)(?P/.*|$)", referer_path).group("address") + + if referer_site_address == site_address: # Referer site address as simple address + return True + elif self.site_manager.resolveDomain(referer_site_address) == site_address: # Referer site address as dns + return True + else: # Invalid referer + return False + diff --git a/plugins/Zeroname/__init__.py b/plugins/Zeroname/__init__.py new file mode 100644 index 00000000..889802db --- /dev/null +++ b/plugins/Zeroname/__init__.py @@ -0,0 +1,2 @@ +import UiRequestPlugin +import SiteManagerPlugin \ No newline at end of file diff --git a/plugins/Zeroname/updater/zeroname_updater.py b/plugins/Zeroname/updater/zeroname_updater.py new file mode 100644 index 00000000..f6df3d51 --- /dev/null +++ b/plugins/Zeroname/updater/zeroname_updater.py @@ -0,0 +1,115 @@ +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +import time, json, os, sys, re + + +def publish(): + print "* Signing..." + os.system("python zeronet.py siteSign %s %s" % (config["site"], config["privatekey"])) + print "* Publishing..." + os.system("python zeronet.py sitePublish %s" % config["site"]) + + + +def processNameOp(domain, value): + if not value.startswith("{"): return False + try: + data = json.loads(value) + except Exception, err: + print "Json load error: %s" % err + return False + if "zeronet" not in data: + print "No zeronet in ", data.keys() + return False + + if "slave" in sys.argv: + print "Waiting for master update arrive" + time.sleep(30) # Wait 30 sec to allow master updater + + names_raw = open(names_path, "rb").read() + names = json.loads(names_raw) + for subdomain, address in data["zeronet"].items(): + print subdomain, domain, "->", address + if subdomain: + names["%s.%s.bit" % (subdomain, domain)] = address + else: + names["%s.bit" % domain] = address + + new_names_raw = json.dumps(names, indent=2) + if new_names_raw != names_raw: + open(names_path, "wb").write(new_names_raw) + return True + else: + print "names not changed" + return False + + + + +def processBlock(block_id): + print "Processing block #%s..." % block_id + block_hash = rpc.getblockhash(block_id) + block = rpc.getblock(block_hash) + + print "Checking %s tx" % len(block["tx"]) + updated = 0 + for tx in block["tx"]: + transaction = rpc.getrawtransaction(tx, 1) + for vout in transaction.get("vout",[]): + if "scriptPubKey" in vout and "nameOp" in vout["scriptPubKey"] and "name" in vout["scriptPubKey"]["nameOp"]: + name_op = vout["scriptPubKey"]["nameOp"] + updated += processNameOp(name_op["name"].replace("d/", ""), name_op["value"]) + print "Done (updated %s)." % updated + if updated: + publish() + + +# Loading config... +config_path = os.path.expanduser("~/.namecoin/zeroname_config.json") +if not os.path.isfile(config_path): # Create sample config + open(config_path, "w").write( + json.dumps({'site': 'site', 'zeronet_path': '/home/zeronet/', 'privatekey': '', 'lastprocessed': None}, indent=2) + ) + print "Example config written to %s" % config_path + sys.exit(0) + +config = json.load(open(config_path)) +names_path = "%s/data/%s/data/names.json" % (config["zeronet_path"], config["site"]) +os.chdir(config["zeronet_path"]) # Change working dir + +# Getting rpc connect details +namecoin_conf = open(os.path.expanduser("~/.namecoin/namecoin.conf")).read() + +# Connecting to RPC +rpc_user = re.search("rpcuser=(.*)$", namecoin_conf, re.M).group(1) +rpc_pass = re.search("rpcpassword=(.*)$", namecoin_conf, re.M).group(1) +rpc_url = "http://%s:%s@127.0.0.1:8336" % (rpc_user, rpc_pass) +rpc = AuthServiceProxy(rpc_url, timeout=60*5) + +last_block = int(rpc.getinfo()["blocks"]) + +if not config["lastprocessed"]: # Start processing from last block + config["lastprocessed"] = last_block + +# Processing skipped blocks +print "Processing block from #%s to #%s..." % (config["lastprocessed"], last_block) +for block_id in range(config["lastprocessed"], last_block+1): + processBlock(block_id) + +#processBlock(223911) # Testing + +while 1: + print "Waiting for new block..." + while 1: + try: + rpc = AuthServiceProxy(rpc_url, timeout=60*5) + if (int(rpc.getinfo()["blocks"]) > last_block): break + time.sleep(1) + rpc.waitforblock() + break # Block found + except Exception, err: # Timeout + pass + last_block = int(rpc.getinfo()["blocks"]) + processBlock(last_block) + + config["lastprocessed"] = last_block + open(config_path, "w").write(json.dumps(config, indent=2)) diff --git a/plugins/disabled-Dnschain/SiteManagerPlugin.py b/plugins/disabled-Dnschain/SiteManagerPlugin.py new file mode 100644 index 00000000..6f757e8d --- /dev/null +++ b/plugins/disabled-Dnschain/SiteManagerPlugin.py @@ -0,0 +1,153 @@ +import logging, json, os, re, sys, time +import gevent +from Plugin import PluginManager +from Config import config +from util import Http +from Debug import Debug + +allow_reload = False # No reload supported + +log = logging.getLogger("DnschainPlugin") + +@PluginManager.registerTo("SiteManager") +class SiteManagerPlugin(object): + dns_cache_path = "data/dns_cache.json" + dns_cache = None + + # Checks if its a valid address + def isAddress(self, address): + if self.isDomain(address): + return True + else: + return super(SiteManagerPlugin, self).isAddress(address) + + + # Return: True if the address is domain + def isDomain(self, address): + return re.match("(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9]+)$", address) + + + # Load dns entries from data/dns_cache.json + def loadDnsCache(self): + if os.path.isfile(self.dns_cache_path): + self.dns_cache = json.load(open(self.dns_cache_path)) + else: + self.dns_cache = {} + log.debug("Loaded dns cache, entries: %s" % len(self.dns_cache)) + + + # Save dns entries to data/dns_cache.json + def saveDnsCache(self): + json.dump(self.dns_cache, open(self.dns_cache_path, "wb"), indent=2) + + + # Resolve domain using dnschain.net + # Return: The address or None + def resolveDomainDnschainNet(self, domain): + try: + match = self.isDomain(domain) + sub_domain = match.group(1).strip(".") + top_domain = match.group(2) + if not sub_domain: sub_domain = "@" + address = None + with gevent.Timeout(5, Exception("Timeout: 5s")): + res = Http.get("https://api.dnschain.net/v1/namecoin/key/%s" % top_domain).read() + data = json.loads(res)["data"]["value"] + if "zeronet" in data: + for key, val in data["zeronet"].iteritems(): + self.dns_cache[key+"."+top_domain] = [val, time.time()+60*60*5] # Cache for 5 hours + self.saveDnsCache() + return data["zeronet"].get(sub_domain) + # Not found + return address + except Exception, err: + log.debug("Dnschain.net %s resolve error: %s" % (domain, Debug.formatException(err))) + + + # Resolve domain using dnschain.info + # Return: The address or None + def resolveDomainDnschainInfo(self, domain): + try: + match = self.isDomain(domain) + sub_domain = match.group(1).strip(".") + top_domain = match.group(2) + if not sub_domain: sub_domain = "@" + address = None + with gevent.Timeout(5, Exception("Timeout: 5s")): + res = Http.get("https://dnschain.info/bit/d/%s" % re.sub("\.bit$", "", top_domain)).read() + data = json.loads(res)["value"] + for key, val in data["zeronet"].iteritems(): + self.dns_cache[key+"."+top_domain] = [val, time.time()+60*60*5] # Cache for 5 hours + self.saveDnsCache() + return data["zeronet"].get(sub_domain) + # Not found + return address + except Exception, err: + log.debug("Dnschain.info %s resolve error: %s" % (domain, Debug.formatException(err))) + + + # Resolve domain + # Return: The address or None + def resolveDomain(self, domain): + domain = domain.lower() + if self.dns_cache == None: + self.loadDnsCache() + if domain.count(".") < 2: # Its a topleved request, prepend @. to it + domain = "@."+domain + + domain_details = self.dns_cache.get(domain) + if domain_details and time.time() < domain_details[1]: # Found in cache and its not expired + return domain_details[0] + else: + # Resovle dns using dnschain + thread_dnschain_info = gevent.spawn(self.resolveDomainDnschainInfo, domain) + thread_dnschain_net = gevent.spawn(self.resolveDomainDnschainNet, domain) + gevent.joinall([thread_dnschain_net, thread_dnschain_info]) # Wait for finish + + if thread_dnschain_info.value and thread_dnschain_net.value: # Booth successfull + if thread_dnschain_info.value == thread_dnschain_net.value: # Same returned value + return thread_dnschain_info.value + else: + log.error("Dns %s missmatch: %s != %s" % (domain, thread_dnschain_info.value, thread_dnschain_net.value)) + + # Problem during resolve + if domain_details: # Resolve failed, but we have it in the cache + domain_details[1] = time.time()+60*60 # Dont try again for 1 hour + return domain_details[0] + else: # Not found in cache + self.dns_cache[domain] = [None, time.time()+60] # Don't check again for 1 min + return None + + + # Return or create site and start download site files + # Return: Site or None if dns resolve failed + def need(self, address, all_file=True): + if self.isDomain(address): # Its looks like a domain + address_resolved = self.resolveDomain(address) + if address_resolved: + address = address_resolved + else: + return None + + return super(SiteManagerPlugin, self).need(address, all_file) + + + # Return: Site object or None if not found + def get(self, address): + if self.sites == None: # Not loaded yet + self.load() + if self.isDomain(address): # Its looks like a domain + address_resolved = self.resolveDomain(address) + if address_resolved: # Domain found + site = self.sites.get(address_resolved) + if site: + site_domain = site.settings.get("domain") + if site_domain != address: + site.settings["domain"] = address + else: # Domain not found + site = self.sites.get(address) + + else: # Access by site address + site = self.sites.get(address) + return site + diff --git a/plugins/disabled-Dnschain/UiRequestPlugin.py b/plugins/disabled-Dnschain/UiRequestPlugin.py new file mode 100644 index 00000000..65a386f1 --- /dev/null +++ b/plugins/disabled-Dnschain/UiRequestPlugin.py @@ -0,0 +1,34 @@ +import re +from Plugin import PluginManager + +@PluginManager.registerTo("UiRequest") +class UiRequestPlugin(object): + def __init__(self, server = None): + from Site import SiteManager + self.site_manager = SiteManager.site_manager + super(UiRequestPlugin, self).__init__(server) + + + # Media request + def actionSiteMedia(self, path): + match = re.match("/media/(?P
[A-Za-z0-9]+\.[A-Za-z0-9\.]+)(?P/.*|$)", path) + if match: # Its a valid domain, resolve first + domain = match.group("address") + address = self.site_manager.resolveDomain(domain) + if address: + path = "/media/"+address+match.group("inner_path") + return super(UiRequestPlugin, self).actionSiteMedia(path) # Get the wrapper frame output + + + # Is mediarequest allowed from that referer + def isMediaRequestAllowed(self, site_address, referer): + referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address + referer_site_address = re.match("/(?P
[A-Za-z0-9\.]+)(?P/.*|$)", referer_path).group("address") + + if referer_site_address == site_address: # Referer site address as simple address + return True + elif self.site_manager.resolveDomain(referer_site_address) == site_address: # Referer site address as dns + return True + else: # Invalid referer + return False + diff --git a/plugins/disabled-Dnschain/__init__.py b/plugins/disabled-Dnschain/__init__.py new file mode 100644 index 00000000..83c9bee0 --- /dev/null +++ b/plugins/disabled-Dnschain/__init__.py @@ -0,0 +1,2 @@ +import DnschainPlugin +import SiteManagerPlugin \ No newline at end of file diff --git a/src/Config.py b/src/Config.py index d3421c19..af49bc78 100644 --- a/src/Config.py +++ b/src/Config.py @@ -3,7 +3,7 @@ import ConfigParser class Config(object): def __init__(self): - self.version = "0.2.7" + self.version = "0.2.8" self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -86,6 +86,7 @@ class Config(object): parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port') + parser.add_argument('--disable_zeromq', help='Disable compatibility with old clients', action='store_true') parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip') diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index d0816446..eb8f9282 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -3,10 +3,14 @@ from cStringIO import StringIO import gevent, msgpack from Config import config from Debug import Debug -try: - import zmq.green as zmq -except: - zmq = None +zmq = None +if not config.disable_zeromq: + try: + import zmq.green as zmq + except: + zmq = None + + class Connection: def __init__(self, server, ip, port, sock=None): @@ -75,7 +79,8 @@ class Connection: try: firstchar = sock.recv(1) # Find out if pure socket or zeromq except Exception, err: - self.log.debug("Socket firstchar error: %s" % Debug.formatException(err)) + if self.log: + self.log.debug("Socket firstchar error: %s" % Debug.formatException(err)) self.close() return False if firstchar == "\xff": # Backward compatiblity: forward data to zmq @@ -106,7 +111,7 @@ class Connection: try: if not firstchar: firstchar = sock.recv(1) except Exception, err: - self.log.debug("Socket firstchar error: %s" % Debug.formatException(err)) + if self.log: self.log.debug("Socket firstchar error: %s" % Debug.formatException(err)) self.close() return False if firstchar == "\xff": # Backward compatibility to zmq @@ -294,3 +299,4 @@ class Connection: del self.log del self.unpacker del self.sock + self.unpacker = None diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 12b42a50..baf3349e 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -26,7 +26,7 @@ class ConnectionServer: self.zmq_last_connection = None # Last incoming message client self.peer_id = "-ZN0"+config.version.replace(".", "")+"-"+''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12)) # Bittorrent style peerid - + if port: # Listen server on a port self.zmq_port = port-1 self.pool = Pool(1000) # do not accept more than 1000 connections @@ -144,6 +144,9 @@ class ConnectionServer: def zmqServer(self): + if config.disable_zeromq: + self.log.debug("ZeroMQ disabled by config") + return False self.log.debug("Starting ZeroMQ on: tcp://127.0.0.1:%s..." % self.zmq_port) try: import zmq.green as zmq diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 8a6c9c6c..0538c527 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -1,6 +1,5 @@ import os, logging, urllib2, re, time import gevent, msgpack -import zmq.green as zmq from Config import config from FileRequest import FileRequest from Site import SiteManager @@ -17,7 +16,7 @@ class FileServer(ConnectionServer): SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist else: self.port_opened = None # Is file server opened on router - self.sites = SiteManager.list() + self.sites = SiteManager.site_manager.list() # Handle request to fileserver diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 85b85733..cc13daad 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -1,5 +1,4 @@ import os, logging, gevent, time, msgpack, sys -import zmq.green as zmq from cStringIO import StringIO from Config import config from Debug import Debug diff --git a/src/Plugin/PluginManager.py b/src/Plugin/PluginManager.py index 6b6f2594..3f050e9c 100644 --- a/src/Plugin/PluginManager.py +++ b/src/Plugin/PluginManager.py @@ -7,6 +7,7 @@ class PluginManager: self.log = logging.getLogger("PluginManager") self.plugin_path = "plugins" # Plugin directory self.plugins = {} # Registered plugins (key: class name, value: list of plugins for class) + self.plugin_names = [] # Loaded plugin names sys.path.append(self.plugin_path) @@ -30,6 +31,7 @@ class PluginManager: __import__(dir_name) except Exception, err: self.log.error("Plugin %s load error: %s" % (dir_name, Debug.formatException(err))) + if dir_name not in self.plugin_names: self.plugin_names.append(dir_name) # Reload all plugins diff --git a/src/Site/Site.py b/src/Site/Site.py index f10ca575..776c1a13 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -181,10 +181,12 @@ class Site: # Publish worker - def publisher(self, inner_path, peers, published, limit): + def publisher(self, inner_path, peers, published, limit, event_done): timeout = 5+int(self.storage.getSize(inner_path)/1024) # Timeout: 5sec + size in kb while 1: - if not peers or len(published) >= limit: break # All peers done, or published engouht + if not peers or len(published) >= limit: + event_done.set(True) + break # All peers done, or published engouht peer = peers.pop(0) result = {"exception": "Timeout"} @@ -216,11 +218,14 @@ class Site: peers = self.peers.values() random.shuffle(peers) - for i in range(limit): - publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit) + event_done = gevent.event.AsyncResult() + for i in range(min(1+len(self.peers), limit)/2): + publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, event_done) publishers.append(publisher) - gevent.joinall(publishers) # Wait for all publishers + event_done.get() # Wait for done + if len(published) < min(len(self.peers), limit): time.sleep(0.2) # If less than we need sleep a bit + if len(published) == 0: gevent.join(publishers) # No successful publish, wait for all publisher self.log.info("Successfuly published to %s peers" % len(published)) return len(published) @@ -233,6 +238,7 @@ class Site: elif self.settings["serving"] == False: # Site not serving return False else: # Wait until file downloaded + self.bad_files[inner_path] = True # Mark as bad file if not self.content_manager.contents.get("content.json"): # No content.json, download it first! self.log.debug("Need content.json first") self.announce() @@ -348,7 +354,7 @@ class Site: def fileDone(self, inner_path): # File downloaded, remove it from bad files if inner_path in self.bad_files: - self.log.info("Bad file solved: %s" % inner_path) + self.log.debug("Bad file solved: %s" % inner_path) del(self.bad_files[inner_path]) # Update content.json last downlad time diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 87d1b0d3..295b744e 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -1,75 +1,90 @@ import json, logging, time, re, os import gevent +from Plugin import PluginManager TRACKERS = [ ("udp", "open.demonii.com", 1337), ("udp", "sugoi.pomf.se", 2710), - ("udp", "tracker.coppersurfer.tk", 80), + #("udp", "tracker.coppersurfer.tk", 80), ("udp", "tracker.leechers-paradise.org", 6969), ("udp", "9.rarbg.com", 2710), - #("udp", "www.eddie4.nl", 6969), Backup trackers - #("udp", "trackr.sytes.net", 80), + ("udp", "www.eddie4.nl", 6969), + #("udp", "trackr.sytes.net", 80), #Backup trackers #("udp", "tracker4.piratux.com", 6969) ] -# Load all sites from data/sites.json -def load(): - from Site import Site - global sites - if not sites: sites = {} - address_found = [] - added = 0 - # Load new adresses - for address in json.load(open("data/sites.json")): - if address not in sites and os.path.isfile("data/%s/content.json" % address): - sites[address] = Site(address) - added += 1 - address_found.append(address) - # Remove deleted adresses - for address in sites.keys(): - if address not in address_found: - del(sites[address]) - logging.debug("Removed site: %s" % address) +@PluginManager.acceptPlugins +class SiteManager(object): + def __init__(self): + self.sites = None - if added: logging.debug("SiteManager added %s sites" % added) + # Load all sites from data/sites.json + def load(self): + from Site import Site + if not self.sites: self.sites = {} + address_found = [] + added = 0 + # Load new adresses + for address in json.load(open("data/sites.json")): + if address not in self.sites and os.path.isfile("data/%s/content.json" % address): + self.sites[address] = Site(address) + added += 1 + address_found.append(address) + + # Remove deleted adresses + for address in self.sites.keys(): + if address not in address_found: + del(self.sites[address]) + logging.debug("Removed site: %s" % address) + + if added: logging.debug("SiteManager added %s sites" % added) -# Checks if its a valid address -def isAddress(address): - return re.match("^[A-Za-z0-9]{26,35}$", address) + # Checks if its a valid address + def isAddress(self, address): + return re.match("^[A-Za-z0-9]{26,35}$", address) -# Return site and start download site files -def need(address, all_file=True): - from Site import Site - new = False - if address not in sites: # Site not exits yet - if not isAddress(address): return False # Not address: %s % address - logging.debug("Added new site: %s" % address) - sites[address] = Site(address) - if not sites[address].settings["serving"]: # Maybe it was deleted before - sites[address].settings["serving"] = True - sites[address].saveSettings() - new = True - - site = sites[address] - if all_file: site.download() - return site + # Return: Site object or None if not found + def get(self, address): + if self.sites == None: # Not loaded yet + self.load() + return self.sites.get(address) -def delete(address): - global sites - logging.debug("SiteManager deleted site: %s" % address) - del(sites[address]) + # Return or create site and start download site files + def need(self, address, all_file=True): + from Site import Site + new = False + site = self.get(address) + if not site: # Site not exits yet + if not self.isAddress(address): return False # Not address: %s % address + logging.debug("Added new site: %s" % address) + site = Site(address) + self.sites[address] = site + if not site.settings["serving"]: # Maybe it was deleted before + site.settings["serving"] = True + site.saveSettings() + new = True + + if all_file: site.download() + return site -# Lazy load sites -def list(): - if sites == None: # Not loaded yet - load() - return sites + def delete(self, address): + logging.debug("SiteManager deleted site: %s" % address) + del(self.sites[address]) -sites = None + # Lazy load sites + def list(self): + if self.sites == None: # Not loaded yet + self.load() + return self.sites + + + +site_manager = SiteManager() # Singletone + peer_blacklist = [] # Dont download from this peers \ No newline at end of file diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 333ee1b2..6a6ce86c 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -10,6 +10,7 @@ status_texts = { 400: "400 Bad Request", 403: "403 Forbidden", 404: "404 Not Found", + 500: "500 Internal Server Error", } @@ -125,27 +126,32 @@ class UiRequest(object): # Render a file from media with iframe site wrapper - def actionWrapper(self, path, extra_headers=[]): - if "." in path and not path.endswith(".html"): return self.actionSiteMedia("/media"+path) # Only serve html files with frame + def actionWrapper(self, path, extra_headers=None): + if not extra_headers: extra_headers = [] if self.get.get("wrapper") == "False": return self.actionSiteMedia("/media"+path) # Only serve html files with frame - if self.env.get("HTTP_X_REQUESTED_WITH"): return self.error403() # No ajax allowed on wrapper - match = re.match("/(?P[A-Za-z0-9]+)(?P/.*|$)", path) + match = re.match("/(?P
[A-Za-z0-9\._-]+)(?P/.*|$)", path) if match: + address = match.group("address") inner_path = match.group("inner_path").lstrip("/") + if "." in inner_path and not inner_path.endswith(".html"): return self.actionSiteMedia("/media"+path) # Only serve html files with frame + if self.env.get("HTTP_X_REQUESTED_WITH"): return self.error403("Ajax request not allowed to load wrapper") # No ajax allowed on wrapper + if not inner_path: inner_path = "index.html" # If inner path defaults to index.html - site = self.server.sites.get(match.group("site")) + site = SiteManager.site_manager.get(address) + if site and site.content_manager.contents.get("content.json") and (not site.getReachableBadFiles() or site.settings["own"]): # Its downloaded or own title = site.content_manager.contents["content.json"]["title"] else: - title = "Loading %s..." % match.group("site") - site = SiteManager.need(match.group("site")) # Start download site + title = "Loading %s..." % address + site = SiteManager.site_manager.need(address) # Start download site + if not site: return False extra_headers.append(("X-Frame-Options", "DENY")) - self.sendHeader(extra_headers=extra_headers) + self.sendHeader(extra_headers=extra_headers[:]) # Wrapper variable inits query_string = "" @@ -162,7 +168,7 @@ class UiRequest(object): return self.render("src/Ui/template/wrapper.html", inner_path=inner_path, - address=match.group("site"), + address=address, title=title, body_style=body_style, meta_tags=meta_tags, @@ -177,33 +183,39 @@ class UiRequest(object): return False + # Returns if media request allowed from that referer + def isMediaRequestAllowed(self, site_address, referer): + referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address + return referer_path.startswith("/"+site_address) + + # Serve a media for site def actionSiteMedia(self, path): path = path.replace("/index.html/", "/") # Base Backward compatibility fix - match = re.match("/media/(?P[A-Za-z0-9]+)/(?P.*)", path) + match = re.match("/media/(?P
[A-Za-z0-9\._-]+)/(?P.*)", path) referer = self.env.get("HTTP_REFERER") - if referer: # Only allow same site to receive media - referer = re.sub("http://.*?/", "/", referer) # Remove server address - referer = referer.replace("/media", "") # Media - if not referer.startswith("/"+match.group("site")): return self.error403() # Referer not starts same address as requested path + if referer and match: # Only allow same site to receive media + if not self.isMediaRequestAllowed(match.group("address"), referer): + return self.error403("Media referer error") # Referer not starts same address as requested path if match: # Looks like a valid path - file_path = "data/%s/%s" % (match.group("site"), match.group("inner_path")) - allowed_dir = os.path.abspath("data/%s" % match.group("site")) # Only files within data/sitehash allowed + address = match.group("address") + file_path = "data/%s/%s" % (address, match.group("inner_path")) + allowed_dir = os.path.abspath("data/%s" % address) # Only files within data/sitehash allowed if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir): # File not in allowed path return self.error403() else: if config.debug and file_path.split("/")[-1].startswith("all."): # When debugging merge *.css to all.css and *.js to all.js - site = self.server.sites.get(match.group("site")) + site = self.server.sites.get(address) if site.settings["own"]: from Debug import DebugMedia DebugMedia.merge(file_path) if os.path.isfile(file_path): # File exits return self.actionFile(file_path) else: # File not exits, try to download - site = SiteManager.need(match.group("site"), all_file=False) + site = SiteManager.site_manager.need(address, all_file=False) self.sendHeader(content_type=self.getContentType(file_path)) # ?? Get Exception without this result = site.needFile(match.group("inner_path"), priority=1) # Wait until file downloads return self.actionFile(file_path) @@ -323,9 +335,9 @@ class UiRequest(object): # You are not allowed to access this - def error403(self): + def error403(self, message="Forbidden"): self.sendHeader(403) - return "Forbidden" + return message # Send file not found error @@ -333,6 +345,13 @@ class UiRequest(object): self.sendHeader(404) return "Not Found: %s" % path + + # Internal server error + def error500(self, message = ":("): + self.sendHeader(500) + return "

Server error

%s" % cgi.escape(message) + + # - Reload for eaiser developing - def reload(self): import imp, sys diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index b677887d..a8bd1f1d 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -1,4 +1,3 @@ -from gevent import monkey; monkey.patch_all(thread = False) import logging, time, cgi, string, random from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIHandler @@ -29,7 +28,7 @@ class UiWSGIHandler(WSGIHandler): try: return super(UiWSGIHandler, self).run_application() except Exception, err: - logging.debug("UiWSGIHandler error: %s" % Debug.formatException(err)) + logging.error("UiWSGIHandler error: %s" % Debug.formatException(err)) if config.debug: # Allow websocket errors to appear on /Debug import sys sys.modules["main"].DebugHook.handleError() @@ -43,7 +42,7 @@ class UiServer: if self.ip == "*": self.ip = "" # Bind all #self.sidebar_websockets = [] # Sidebar websocket connections #self.auth_key = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(12)) # Global admin auth key - self.sites = SiteManager.list() + self.sites = SiteManager.site_manager.list() self.log = logging.getLogger(__name__) self.ui_request = UiRequest(self) @@ -58,8 +57,14 @@ class UiServer: self.ui_request.get = dict(cgi.parse_qsl(env['QUERY_STRING'])) else: self.ui_request.get = {} - return self.ui_request.route(path) - + if config.debug: # Let the exception catched by werkezung + return self.ui_request.route(path) + else: # Catch and display the error + try: + return self.ui_request.route(path) + except Exception, err: + logging.debug("UiRequest error: %s" % Debug.formatException(err)) + return self.ui_request.error500("Err: %s" % Debug.formatException(err)) # Reload the UiRequest class to prevent restarts in debug mode def reload(self): diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 602692ab..fd5927be 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -220,7 +220,8 @@ class UiWebsocket(object): "ui_ip": config.ui_ip, "ui_port": config.ui_port, "version": config.version, - "debug": config.debug + "debug": config.debug, + "plugins": PluginManager.plugin_manager.plugin_names } @@ -327,10 +328,10 @@ class UiWebsocket(object): # List all site info def actionSiteList(self, to): ret = [] - SiteManager.load() # Reload sites + SiteManager.site_manager.load() # Reload sites for site in self.server.sites.values(): if not site.content_manager.contents.get("content.json"): continue # Broken site - ret.append(self.formatSiteInfo(site, create_user=False)) + ret.append(self.formatSiteInfo(site, create_user=False)) # Dont generate the auth_address on listing self.response(to, ret) @@ -386,7 +387,7 @@ class UiWebsocket(object): site.worker_manager.running = False site.worker_manager.stopWorkers() site.storage.deleteFiles() - SiteManager.delete(address) + SiteManager.site_manager.delete(address) site.updateWebsocket() else: self.response(to, {"error": "Unknown site: %s" % address}) diff --git a/src/Ui/media/Loading.coffee b/src/Ui/media/Loading.coffee index 86fbda3d..ca2bf354 100644 --- a/src/Ui/media/Loading.coffee +++ b/src/Ui/media/Loading.coffee @@ -7,7 +7,8 @@ class Loading $(".progressbar").css("width", percent*100+"%").css("opacity", "1").css("display", "block") hideProgress: -> - $(".progressbar").css("width", "100%").css("opacity", "0").cssLater("display", "none", 1000) + console.log "hideProgress" + $(".progressbar").css("width", "100%").css("opacity", "0").hideLater(1000) showScreen: -> diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index e451f851..8f7d41a0 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -17,10 +17,12 @@ class Wrapper @ws_error = null # Ws error message @site_info = null # Hold latest site info + @event_site_info = $.Deferred() # Event when site_info received @inner_loaded = false # If iframe loaded or not @inner_ready = false # Inner frame ready to receive messages @wrapperWsInited = false # Wrapper notified on websocket open @site_error = null # Latest failed file download + @address = null window.onload = @onLoad # On iframe loaded $(window).on "hashchange", => # On hash change @@ -47,7 +49,7 @@ class Wrapper @ws.response message.id, res else if cmd == "setSiteInfo" @sendInner message # Pass to inner frame - if message.params.address == window.address # Current page + if message.params.address == @address # Current page @setSiteInfo message.params else if cmd == "updating" # Close connection @ws.ws.close() @@ -159,13 +161,14 @@ class Wrapper actionGetLocalStorage: (message) -> - data = localStorage.getItem "site.#{window.address}" - if data then data = JSON.parse(data) - @sendInner {"cmd": "response", "to": message.id, "result": data} + $.when(@event_site_info).done => + data = localStorage.getItem "site.#{@site_info.address}" + if data then data = JSON.parse(data) + @sendInner {"cmd": "response", "to": message.id, "result": data} actionSetLocalStorage: (message) -> - back = localStorage.setItem "site.#{window.address}", JSON.stringify(message.params) + back = localStorage.setItem "site.#{@site_info.address}", JSON.stringify(message.params) # EOF actions @@ -221,7 +224,9 @@ class Wrapper # Get site info from UiServer reloadSiteInfo: -> @ws.cmd "siteInfo", {}, (site_info) => + @address = site_info.address @setSiteInfo site_info + window.document.title = site_info.content.title+" - ZeroNet" @log "Setting title to", window.document.title @@ -282,6 +287,7 @@ class Wrapper @loading.hideProgress() @site_info = site_info + @event_site_info.resolve() toHtmlSafe: (values) -> diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index c1eba6a5..c473e846 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -214,7 +214,9 @@ jQuery.fx.step.scale = function(fx) { } elem = this; setTimeout((function() { - return elem.css("display", "none"); + if (elem.css("opacity") === 0) { + return elem.css("display", "none"); + } }), time); return this; }; @@ -474,7 +476,8 @@ jQuery.extend( jQuery.easing, }; Loading.prototype.hideProgress = function() { - return $(".progressbar").css("width", "100%").css("opacity", "0").cssLater("display", "none", 1000); + console.log("hideProgress"); + return $(".progressbar").css("width", "100%").css("opacity", "0").hideLater(1000); }; Loading.prototype.showScreen = function() { @@ -660,7 +663,6 @@ jQuery.extend( jQuery.easing, }).call(this); - /* ---- src/Ui/media/Sidebar.coffee ---- */ @@ -756,10 +758,12 @@ jQuery.extend( jQuery.easing, this.ws.connect(); this.ws_error = null; this.site_info = null; + this.event_site_info = $.Deferred(); this.inner_loaded = false; this.inner_ready = false; this.wrapperWsInited = false; this.site_error = null; + this.address = null; window.onload = this.onLoad; $(window).on("hashchange", (function(_this) { return function() { @@ -794,7 +798,7 @@ jQuery.extend( jQuery.easing, })(this)); } else if (cmd === "setSiteInfo") { this.sendInner(message); - if (message.params.address === window.address) { + if (message.params.address === this.address) { return this.setSiteInfo(message.params); } } else if (cmd === "updating") { @@ -947,21 +951,25 @@ jQuery.extend( jQuery.easing, }; Wrapper.prototype.actionGetLocalStorage = function(message) { - var data; - data = localStorage.getItem("site." + window.address); - if (data) { - data = JSON.parse(data); - } - return this.sendInner({ - "cmd": "response", - "to": message.id, - "result": data - }); + return $.when(this.event_site_info).done((function(_this) { + return function() { + var data; + data = localStorage.getItem("site." + _this.site_info.address); + if (data) { + data = JSON.parse(data); + } + return _this.sendInner({ + "cmd": "response", + "to": message.id, + "result": data + }); + }; + })(this)); }; Wrapper.prototype.actionSetLocalStorage = function(message) { var back; - return back = localStorage.setItem("site." + window.address, JSON.stringify(message.params)); + return back = localStorage.setItem("site." + this.site_info.address, JSON.stringify(message.params)); }; Wrapper.prototype.onOpenWebsocket = function(e) { @@ -1032,6 +1040,7 @@ jQuery.extend( jQuery.easing, Wrapper.prototype.reloadSiteInfo = function() { return this.ws.cmd("siteInfo", {}, (function(_this) { return function(site_info) { + _this.address = site_info.address; _this.setSiteInfo(site_info); window.document.title = site_info.content.title + " - ZeroNet"; return _this.log("Setting title to", window.document.title); @@ -1108,7 +1117,8 @@ jQuery.extend( jQuery.easing, } else { this.loading.hideProgress(); } - return this.site_info = site_info; + this.site_info = site_info; + return this.event_site_info.resolve(); }; Wrapper.prototype.toHtmlSafe = function(values) { @@ -1154,4 +1164,4 @@ jQuery.extend( jQuery.easing, window.wrapper = new Wrapper(ws_url); -}).call(this); \ No newline at end of file +}).call(this); diff --git a/src/Ui/media/lib/jquery.csslater.coffee b/src/Ui/media/lib/jquery.csslater.coffee index 43c94149..ec266622 100644 --- a/src/Ui/media/lib/jquery.csslater.coffee +++ b/src/Ui/media/lib/jquery.csslater.coffee @@ -16,7 +16,8 @@ jQuery.fn.removeLater = (time = 500) -> jQuery.fn.hideLater = (time = 500) -> elem = @ setTimeout ( -> - elem.css("display", "none") + if elem.css("opacity") == 0 + elem.css("display", "none") ), time return @ diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index 6f991ed4..19becec1 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -156,7 +156,7 @@ class WorkerManager: peers = None task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_added": time.time(), "time_started": None, "peers": peers, "priority": priority, "failed": []} self.tasks.append(task) - self.started_task_num = len(self.tasks) + self.started_task_num += 1 self.log.debug("New task: %s, peer lock: %s, priority: %s, tasks: %s" % (task["inner_path"], peers, priority, self.started_task_num)) self.startWorkers(peers) return evt @@ -176,6 +176,8 @@ class WorkerManager: self.tasks.remove(task) # Remove from queue self.site.onFileFail(task["inner_path"]) task["evt"].set(False) + if not self.tasks: + self.started_task_num = 0 # Mark a task done @@ -184,5 +186,7 @@ class WorkerManager: self.tasks.remove(task) # Remove from queue self.site.onFileDone(task["inner_path"]) task["evt"].set(True) - if not self.tasks: self.site.onComplete() # No more task trigger site complete + if not self.tasks: + self.started_task_num = 0 + self.site.onComplete() # No more task trigger site complete diff --git a/src/main.py b/src/main.py index ade19447..99dfd0a6 100644 --- a/src/main.py +++ b/src/main.py @@ -41,7 +41,6 @@ else: console_log.setLevel(logging.INFO) # Display only important info to console from gevent import monkey; monkey.patch_all() # Make time, thread, socket gevent compatible - import gevent import time diff --git a/src/util/GeventSslPatch.py b/src/util/GeventSslPatch.py new file mode 100644 index 00000000..5356b5f4 --- /dev/null +++ b/src/util/GeventSslPatch.py @@ -0,0 +1,49 @@ +# Re-add sslwrap to Python 2.7.9 +# https://github.com/gevent/gevent/issues/477 + +import inspect +__ssl__ = __import__('ssl') + +try: + _ssl = __ssl__._ssl +except AttributeError: + _ssl = __ssl__._ssl2 + + +OldSSLSocket = __ssl__.SSLSocket + +class NewSSLSocket(OldSSLSocket): + """Fix SSLSocket constructor.""" + def __init__( + self, sock, keyfile=None, certfile=None, server_side=False, cert_reqs=0, + ssl_version=2, ca_certs=None, do_handshake_on_connect=True, + suppress_ragged_eofs=True, ciphers=None, + server_hostname=None, _context=None + ): + OldSSLSocket.__init__( + self, sock, keyfile=None, certfile=None, server_side=False, cert_reqs=0, + ssl_version=2, ca_certs=None, do_handshake_on_connect=True, + suppress_ragged_eofs=True, ciphers=None + ) + + +def new_sslwrap( + sock, server_side=False, keyfile=None, certfile=None, + cert_reqs=__ssl__.CERT_NONE, ssl_version=__ssl__.PROTOCOL_SSLv23, + ca_certs=None, ciphers=None +): + context = __ssl__.SSLContext(ssl_version) + context.verify_mode = cert_reqs or __ssl__.CERT_NONE + if ca_certs: + context.load_verify_locations(ca_certs) + if certfile: + context.load_cert_chain(certfile, keyfile) + if ciphers: + context.set_ciphers(ciphers) + + caller_self = inspect.currentframe().f_back.f_locals['self'] + return context._wrap_socket(sock, server_side=server_side, ssl_sock=caller_self) + +if not hasattr(_ssl, 'sslwrap'): + _ssl.sslwrap = new_sslwrap + __ssl__.SSLSocket = NewSSLSocket \ No newline at end of file diff --git a/src/util/Http.py b/src/util/Http.py new file mode 100644 index 00000000..62d4bc73 --- /dev/null +++ b/src/util/Http.py @@ -0,0 +1,11 @@ +import urllib2, logging +import GeventSslPatch +from Config import config + +def get(url): + logging.debug("Get %s" % url) + req = urllib2.Request(url) + req.add_header('User-Agent', "ZeroNet %s (https://github.com/HelloZeroNet/ZeroNet)" % config.version) + req.add_header('Accept', 'application/json') + return urllib2.urlopen(req) +