import re import os import cgi import sys import math import time import json try: import cStringIO as StringIO except: import StringIO import gevent from Config import config from Plugin import PluginManager from Debug import Debug from Translate import Translate from util import helper from ZipStream import ZipStream plugin_dir = "plugins/Sidebar" media_dir = plugin_dir + "/media" sys.path.append(plugin_dir) # To able to load geoip lib loc_cache = {} if "_" not in locals(): _ = Translate(plugin_dir + "/languages/") @PluginManager.registerTo("UiRequest") class UiRequestPlugin(object): # Inject our resources to end of original file streams def actionUiMedia(self, path): if path == "/uimedia/all.js" or path == "/uimedia/all.css": # First yield the original file and header body_generator = super(UiRequestPlugin, self).actionUiMedia(path) for part in body_generator: yield part # Append our media file to the end ext = re.match(".*(js|css)$", path).group(1) plugin_media_file = "%s/all.%s" % (media_dir, ext) if config.debug: # If debugging merge *.css to all.css and *.js to all.js from Debug import DebugMedia DebugMedia.merge(plugin_media_file) if ext == "js": yield _.translateData(open(plugin_media_file).read()) else: for part in self.actionFile(plugin_media_file, send_header=False): yield part elif path.startswith("/uimedia/globe/"): # Serve WebGL globe files file_name = re.match(".*/(.*)", path).group(1) plugin_media_file = "%s-globe/%s" % (media_dir, file_name) if config.debug and path.endswith("all.js"): # If debugging merge *.css to all.css and *.js to all.js from Debug import DebugMedia DebugMedia.merge(plugin_media_file) for part in self.actionFile(plugin_media_file): yield part else: for part in super(UiRequestPlugin, self).actionUiMedia(path): yield part def actionZip(self): address = self.get["address"] site = self.server.site_manager.get(address) if not site: return self.error404("Site not found") title = site.content_manager.contents.get("content.json", {}).get("title", "").encode('ascii', 'ignore') filename = "%s-backup-%s.zip" % (title, time.strftime("%Y-%m-%d_%H_%M")) self.sendHeader(content_type="application/zip", extra_headers={'Content-Disposition': 'attachment; filename="%s"' % filename}) return self.streamZip(site.storage.getPath(".")) def streamZip(self, file_path): zs = ZipStream(file_path) while 1: data = zs.read() if not data: break yield data @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): def sidebarRenderPeerStats(self, body, site): connected = len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]) connectable = len([peer_id for peer_id in site.peers.keys() if not peer_id.endswith(":0")]) onion = len([peer_id for peer_id in site.peers.keys() if ".onion" in peer_id]) local = len([peer for peer in site.peers.values() if helper.isPrivateIp(peer.ip)]) peers_total = len(site.peers) # Add myself if site.settings["serving"]: peers_total += 1 if site.connection_server.port_opened: connectable += 1 if site.connection_server.tor_manager.start_onions: onion += 1 if peers_total: percent_connected = float(connected) / peers_total percent_connectable = float(connectable) / peers_total percent_onion = float(onion) / peers_total else: percent_connectable = percent_connected = percent_onion = 0 if local: local_html = _(u"<li class='color-yellow'><span>{_[Local]}:</span><b>{local}</b></li>") else: local_html = "" peer_ips = [peer.key for peer in site.getConnectablePeers(20, allow_private=False)] peer_ips.sort(key=lambda peer_ip: ".onion:" in peer_ip) copy_link = "http://127.0.0.1:43110/%s/?zeronet_peers=%s" % ( site.content_manager.contents["content.json"].get("domain", site.address), ",".join(peer_ips) ) body.append(_(u""" <li> <label> {_[Peers]} <small class="label-right"><a href='{copy_link}' id='link-copypeers' class='link-right'>{_[Copy to clipboard]}</a></small> </label> <ul class='graph'> <li style='width: 100%' class='total back-black' title="{_[Total peers]}"></li> <li style='width: {percent_connectable:.0%}' class='connectable back-blue' title='{_[Connectable peers]}'></li> <li style='width: {percent_onion:.0%}' class='connected back-purple' title='{_[Onion]}'></li> <li style='width: {percent_connected:.0%}' class='connected back-green' title='{_[Connected peers]}'></li> </ul> <ul class='graph-legend'> <li class='color-green'><span>{_[Connected]}:</span><b>{connected}</b></li> <li class='color-blue'><span>{_[Connectable]}:</span><b>{connectable}</b></li> <li class='color-purple'><span>{_[Onion]}:</span><b>{onion}</b></li> {local_html} <li class='color-black'><span>{_[Total]}:</span><b>{peers_total}</b></li> </ul> </li> """.replace("{local_html}", local_html))) def sidebarRenderTransferStats(self, body, site): recv = float(site.settings.get("bytes_recv", 0)) / 1024 / 1024 sent = float(site.settings.get("bytes_sent", 0)) / 1024 / 1024 transfer_total = recv + sent if transfer_total: percent_recv = recv / transfer_total percent_sent = sent / transfer_total else: percent_recv = 0.5 percent_sent = 0.5 body.append(_(u""" <li> <label>{_[Data transfer]}</label> <ul class='graph graph-stacked'> <li style='width: {percent_recv:.0%}' class='received back-yellow' title="{_[Received bytes]}"></li> <li style='width: {percent_sent:.0%}' class='sent back-green' title="{_[Sent bytes]}"></li> </ul> <ul class='graph-legend'> <li class='color-yellow'><span>{_[Received]}:</span><b>{recv:.2f}MB</b></li> <li class='color-green'<span>{_[Sent]}:</span><b>{sent:.2f}MB</b></li> </ul> </li> """)) def sidebarRenderFileStats(self, body, site): body.append(_(u""" <li> <label> {_[Files]} <small class="label-right"><a href='#Site+directory' id='link-directory' class='link-right'>{_[Open site directory]}</a> <a href='/ZeroNet-Internal/Zip?address={site.address}' id='link-zip' class='link-right' download='site.zip'>{_[Save as .zip]}</a></small> </label> <ul class='graph graph-stacked'> """)) extensions = ( ("html", "yellow"), ("css", "orange"), ("js", "purple"), ("Image", "green"), ("json", "darkblue"), ("User data", "blue"), ("Other", "white"), ("Total", "black") ) # Collect stats size_filetypes = {} size_total = 0 contents = site.content_manager.listContents() # Without user files for inner_path in contents: content = site.content_manager.contents[inner_path] if "files" not in content or content["files"] is None: continue for file_name, file_details in content["files"].items(): size_total += file_details["size"] ext = file_name.split(".")[-1] size_filetypes[ext] = size_filetypes.get(ext, 0) + file_details["size"] # Get user file sizes size_user_content = site.content_manager.contents.execute( "SELECT SUM(size) + SUM(size_files) AS size FROM content WHERE ?", {"not__inner_path": contents} ).fetchone()["size"] if not size_user_content: size_user_content = 0 size_filetypes["User data"] = size_user_content size_total += size_user_content # The missing difference is content.json sizes if "json" in size_filetypes: size_filetypes["json"] += max(0, site.settings["size"] - size_total) size_total = size_other = site.settings["size"] # Bar for extension, color in extensions: if extension == "Total": continue if extension == "Other": size = max(0, size_other) elif extension == "Image": size = size_filetypes.get("jpg", 0) + size_filetypes.get("png", 0) + size_filetypes.get("gif", 0) size_other -= size else: size = size_filetypes.get(extension, 0) size_other -= size if size_total == 0: percent = 0 else: percent = 100 * (float(size) / size_total) percent = math.floor(percent * 100) / 100 # Floor to 2 digits body.append( u"""<li style='width: %.2f%%' class='%s back-%s' title="%s"></li>""" % (percent, _[extension], color, _[extension]) ) # Legend body.append("</ul><ul class='graph-legend'>") for extension, color in extensions: if extension == "Other": size = max(0, size_other) elif extension == "Image": size = size_filetypes.get("jpg", 0) + size_filetypes.get("png", 0) + size_filetypes.get("gif", 0) elif extension == "Total": size = size_total else: size = size_filetypes.get(extension, 0) if extension == "js": title = "javascript" else: title = extension if size > 1024 * 1024 * 10: # Format as mB is more than 10mB size_formatted = "%.0fMB" % (size / 1024 / 1024) else: size_formatted = "%.0fkB" % (size / 1024) body.append(u"<li class='color-%s'><span>%s:</span><b>%s</b></li>" % (color, _[title], size_formatted)) body.append("</ul></li>") def sidebarRenderSizeLimit(self, body, site): free_space = helper.getFreeSpace() / 1024 / 1024 size = float(site.settings["size"]) / 1024 / 1024 size_limit = site.getSizeLimit() percent_used = size / size_limit body.append(_(u""" <li> <label>{_[Size limit]} <small>({_[limit used]}: {percent_used:.0%}, {_[free space]}: {free_space:,d}MB)</small></label> <input type='text' class='text text-num' value="{size_limit}" id='input-sitelimit'/><span class='text-post'>MB</span> <a href='#Set' class='button' id='button-sitelimit'>{_[Set]}</a> </li> """)) def sidebarRenderOptionalFileStats(self, body, site): size_total = float(site.settings["size_optional"]) size_downloaded = float(site.settings["optional_downloaded"]) if not size_total: return False percent_downloaded = size_downloaded / size_total size_formatted_total = size_total / 1024 / 1024 size_formatted_downloaded = size_downloaded / 1024 / 1024 body.append(_(u""" <li> <label>{_[Optional files]}</label> <ul class='graph'> <li style='width: 100%' class='total back-black' title="{_[Total size]}"></li> <li style='width: {percent_downloaded:.0%}' class='connected back-green' title='{_[Downloaded files]}'></li> </ul> <ul class='graph-legend'> <li class='color-green'><span>{_[Downloaded]}:</span><b>{size_formatted_downloaded:.2f}MB</b></li> <li class='color-black'><span>{_[Total]}:</span><b>{size_formatted_total:.2f}MB</b></li> </ul> </li> """)) return True def sidebarRenderOptionalFileSettings(self, body, site): if self.site.settings.get("autodownloadoptional"): checked = "checked='checked'" else: checked = "" body.append(_(u""" <li> <label>{_[Download and help distribute all files]}</label> <input type="checkbox" class="checkbox" id="checkbox-autodownloadoptional" {checked}/><div class="checkbox-skin"></div> """)) autodownload_bigfile_size_limit = int(site.settings.get("autodownload_bigfile_size_limit", config.autodownload_bigfile_size_limit)) body.append(_(u""" <div class='settings-autodownloadoptional'> <label>{_[Auto download big file size limit]}</label> <input type='text' class='text text-num' value="{autodownload_bigfile_size_limit}" id='input-autodownload_bigfile_size_limit'/><span class='text-post'>MB</span> <a href='#Set' class='button' id='button-autodownload_bigfile_size_limit'>{_[Set]}</a> </div> """)) body.append("</li>") def sidebarRenderBadFiles(self, body, site): body.append(_(u""" <li> <label>{_[Needs to be updated]}:</label> <ul class='filelist'> """)) i = 0 for bad_file, tries in site.bad_files.iteritems(): i += 1 body.append(_(u"""<li class='color-red' title="{bad_file_path} ({tries})">{bad_filename}</li>""", { "bad_file_path": bad_file, "bad_filename": helper.getFilename(bad_file), "tries": _.pluralize(tries, "{} try", "{} tries") })) if i > 30: break if len(site.bad_files) > 30: num_bad_files = len(site.bad_files) - 30 body.append(_(u"""<li class='color-red'>{_[+ {num_bad_files} more]}</li>""", nested=True)) body.append(""" </ul> </li> """) def sidebarRenderDbOptions(self, body, site): if site.storage.db: inner_path = site.storage.getInnerPath(site.storage.db.db_path) size = float(site.storage.getSize(inner_path)) / 1024 feeds = len(site.storage.db.schema.get("feeds", {})) else: inner_path = _[u"No database found"] size = 0.0 feeds = 0 body.append(_(u""" <li> <label>{_[Database]} <small>({size:.2f}kB, {_[search feeds]}: {_[{feeds} query]})</small></label> <div class='flex'> <input type='text' class='text disabled' value="{inner_path}" disabled='disabled'/> <a href='#Reload' id="button-dbreload" class='button'>{_[Reload]}</a> <a href='#Rebuild' id="button-dbrebuild" class='button'>{_[Rebuild]}</a> </div> </li> """, nested=True)) def sidebarRenderIdentity(self, body, site): auth_address = self.user.getAuthAddress(self.site.address) rules = self.site.content_manager.getRules("data/users/%s/content.json" % auth_address) if rules and rules.get("max_size"): quota = rules["max_size"] / 1024 try: content = site.content_manager.contents["data/users/%s/content.json" % auth_address] used = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) except: used = 0 used = used / 1024 else: quota = used = 0 body.append(_(u""" <li> <label>{_[Identity address]} <small>({_[limit used]}: {used:.2f}kB / {quota:.2f}kB)</small></label> <div class='flex'> <span class='input text disabled'>{auth_address}</span> <a href='#Change' class='button' id='button-identity'>{_[Change]}</a> </div> </li> """)) def sidebarRenderControls(self, body, site): auth_address = self.user.getAuthAddress(self.site.address) if self.site.settings["serving"]: class_pause = "" class_resume = "hidden" else: class_pause = "hidden" class_resume = "" body.append(_(u""" <li> <label>{_[Site control]}</label> <a href='#Update' class='button noupdate' id='button-update'>{_[Update]}</a> <a href='#Pause' class='button {class_pause}' id='button-pause'>{_[Pause]}</a> <a href='#Resume' class='button {class_resume}' id='button-resume'>{_[Resume]}</a> <a href='#Delete' class='button noupdate' id='button-delete'>{_[Delete]}</a> </li> """)) donate_key = site.content_manager.contents.get("content.json", {}).get("donate", True) site_address = self.site.address body.append(_(u""" <li> <label>{_[Site address]}</label><br> <div class='flex'> <span class='input text disabled'>{site_address}</span> """)) if donate_key == False or donate_key == "": pass elif (type(donate_key) == str or type(donate_key) == unicode) and len(donate_key) > 0: body.append(_(u""" </div> </li> <li> <label>{_[Donate]}</label><br> <div class='flex'> {donate_key} """)) else: body.append(_(u""" <a href='bitcoin:{site_address}' class='button' id='button-donate'>{_[Donate]}</a> """)) body.append(_(u""" </div> </li> """)) def sidebarRenderOwnedCheckbox(self, body, site): if self.site.settings["own"]: checked = "checked='checked'" else: checked = "" body.append(_(u""" <h2 class='owned-title'>{_[This is my site]}</h2> <input type="checkbox" class="checkbox" id="checkbox-owned" {checked}/><div class="checkbox-skin"></div> """)) def sidebarRenderOwnSettings(self, body, site): title = site.content_manager.contents.get("content.json", {}).get("title", "") description = site.content_manager.contents.get("content.json", {}).get("description", "") body.append(_(u""" <li> <label for='settings-title'>{_[Site title]}</label> <input type='text' class='text' value="{title}" id='settings-title'/> </li> <li> <label for='settings-description'>{_[Site description]}</label> <input type='text' class='text' value="{description}" id='settings-description'/> </li> <li> <a href='#Save' class='button' id='button-settings'>{_[Save site settings]}</a> </li> """)) def sidebarRenderContents(self, body, site): has_privatekey = bool(self.user.getSiteData(site.address).get("privatekey")) if has_privatekey: tag_privatekey = _(u"{_[Private key saved.]} <a href='#Forgot+private+key' id='privatekey-forgot' class='link-right'>{_[Forgot]}</a>") else: tag_privatekey = _(u"<a href='#Add+private+key' id='privatekey-add' class='link-right'>{_[Add saved private key]}</a>") body.append(_(u""" <li> <label>{_[Content publishing]} <small class='label-right'>{tag_privatekey}</small></label> """.replace("{tag_privatekey}", tag_privatekey))) # Choose content you want to sign body.append(_(u""" <div class='flex'> <input type='text' class='text' value="content.json" id='input-contents'/> <a href='#Sign-and-Publish' id='button-sign-publish' class='button'>{_[Sign and publish]}</a> <a href='#Sign-or-Publish' id='menu-sign-publish'>\u22EE</a> </div> """)) contents = ["content.json"] contents += site.content_manager.contents.get("content.json", {}).get("includes", {}).keys() body.append(_(u"<div class='contents'>{_[Choose]}: ")) for content in contents: body.append(_("<a href='{content}' class='contents-content'>{content}</a> ")) body.append("</div>") body.append("</li>") def actionSidebarGetHtmlTag(self, to): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") site = self.site body = [] body.append("<div>") body.append("<a href='#Close' class='close'>×</a>") body.append("<h1>%s</h1>" % cgi.escape(site.content_manager.contents.get("content.json", {}).get("title", ""), True)) body.append("<div class='globe loading'></div>") body.append("<ul class='fields'>") self.sidebarRenderPeerStats(body, site) self.sidebarRenderTransferStats(body, site) self.sidebarRenderFileStats(body, site) self.sidebarRenderSizeLimit(body, site) has_optional = self.sidebarRenderOptionalFileStats(body, site) if has_optional: self.sidebarRenderOptionalFileSettings(body, site) self.sidebarRenderDbOptions(body, site) self.sidebarRenderIdentity(body, site) self.sidebarRenderControls(body, site) if site.bad_files: self.sidebarRenderBadFiles(body, site) self.sidebarRenderOwnedCheckbox(body, site) body.append("<div class='settings-owned'>") self.sidebarRenderOwnSettings(body, site) self.sidebarRenderContents(body, site) body.append("</div>") body.append("</ul>") body.append("</div>") body.append("<div class='menu template'>") body.append("<a href='#'' class='menu-item template'>Template</a>") body.append("</div>") self.response(to, "".join(body)) def downloadGeoLiteDb(self, db_path): import urllib import gzip import shutil from util import helper self.log.info("Downloading GeoLite2 City database...") self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0]) db_urls = [ "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz" ] for db_url in db_urls: try: # Download response = helper.httpRequest(db_url) data_size = response.getheader('content-length') data_recv = 0 data = StringIO.StringIO() while True: buff = response.read(1024 * 512) if not buff: break data.write(buff) data_recv += 1024 * 512 if data_size: progress = int(float(data_recv) / int(data_size) * 100) self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress]) self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell()) data.seek(0) # Unpack with gzip.GzipFile(fileobj=data) as gzip_file: shutil.copyfileobj(gzip_file, open(db_path, "wb")) self.cmd("progress", ["geolite-info", _["GeoLite2 City database downloaded!"], 100]) time.sleep(2) # Wait for notify animation return True except Exception as err: self.log.error("Error downloading %s: %s" % (db_url, err)) pass self.cmd("progress", [ "geolite-info", _["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"].format(err, db_urls[0]), -100 ]) def getLoc(self, geodb, ip): global loc_cache if ip in loc_cache: return loc_cache[ip] else: try: loc_data = geodb.get(ip) except: loc_data = None if not loc_data or "location" not in loc_data: loc_cache[ip] = None return None loc = { "lat": loc_data["location"]["latitude"], "lon": loc_data["location"]["longitude"], } if "city" in loc_data: loc["city"] = loc_data["city"]["names"]["en"] if "country" in loc_data: loc["country"] = loc_data["country"]["names"]["en"] loc_cache[ip] = loc return loc def getPeerLocations(self, peers): import maxminddb db_path = config.data_dir + '/GeoLite2-City.mmdb' if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: if not self.downloadGeoLiteDb(db_path): return False geodb = maxminddb.open_database(db_path) peers = peers.values() # Place bars peer_locations = [] placed = {} # Already placed bars here for peer in peers: # Height of bar if peer.connection and peer.connection.last_ping_delay: ping = round(peer.connection.last_ping_delay * 1000) else: ping = None loc = self.getLoc(geodb, peer.ip) if not loc: continue # Create position array lat, lon = loc["lat"], loc["lon"] latlon = "%s,%s" % (lat, lon) if latlon in placed: # Dont place more than 1 bar to same place, fake repos using ip address last two part lat += float(128 - int(peer.ip.split(".")[-2])) / 50 lon += float(128 - int(peer.ip.split(".")[-1])) / 50 latlon = "%s,%s" % (lat, lon) placed[latlon] = True peer_location = {} peer_location.update(loc) peer_location["lat"] = lat peer_location["lon"] = lon peer_location["ping"] = ping peer_locations.append(peer_location) # Append myself my_loc = self.getLoc(geodb, config.ip_external) if my_loc: my_loc["ping"] = 0 peer_locations.append(my_loc) return peer_locations def actionSidebarGetPeers(self, to): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") try: peer_locations = self.getPeerLocations(self.site.peers) globe_data = [] ping_times = [ peer_location["ping"] for peer_location in peer_locations if peer_location["ping"] ] if ping_times: ping_avg = sum(ping_times) / float(len(ping_times)) else: ping_avg = 0 for peer_location in peer_locations: if peer_location["ping"] == 0: # Me height = -0.135 elif peer_location["ping"]: height = min(0.20, math.log(1 + peer_location["ping"] / ping_avg, 300)) else: height = -0.03 globe_data += [peer_location["lat"], peer_location["lon"], height] self.response(to, globe_data) except Exception, err: self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err)) self.response(to, {"error": err}) def actionSiteSetOwned(self, to, owned): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") if self.site.address == config.updatesite: return self.response(to, "You can't change the ownership of the updater site") self.site.settings["own"] = bool(owned) self.site.updateWebsocket(owned=owned) def actionUserSetSitePrivatekey(self, to, privatekey): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") site_data = self.user.sites[self.site.address] site_data["privatekey"] = privatekey self.site.updateWebsocket(set_privatekey=bool(privatekey)) return "ok" def actionSiteSetAutodownloadoptional(self, to, owned): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") self.site.settings["autodownloadoptional"] = bool(owned) self.site.bad_files = {} gevent.spawn(self.site.update, check_files=True) self.site.worker_manager.removeSolvedFileTasks() def actionDbReload(self, to): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") self.site.storage.closeDb() self.site.storage.getDb() return self.response(to, "ok") def actionDbRebuild(self, to): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") self.site.storage.rebuildDb() return self.response(to, "ok")