import time
import html
import os
import json
from collections import OrderedDict
from Plugin import PluginManager
from Config import config
from util import helper
from Debug import Debug
from Db import Db
@PluginManager.registerTo("UiRequest")
class UiRequestPlugin(object):
def formatTableRow(self, row, class_name=""):
back = []
for format, val in row:
if val is None:
formatted = "n/a"
elif format == "since":
if val:
formatted = "%.0f" % (time.time() - val)
else:
formatted = "n/a"
else:
formatted = format % val
back.append("<td>%s</td>" % formatted)
return "<tr class='%s'>%s</tr>" % (class_name, "".join(back))
def getObjSize(self, obj, hpy=None):
if hpy:
return float(hpy.iso(obj).domisize) / 1024
else:
return 0
# /Stats entry point
@helper.encodeResponse
def actionStats(self):
import gc
import sys
from Ui import UiRequest
from Crypt import CryptConnection
import main
hpy = None
if self.get.get("size") == "1": # Calc obj size
try:
import guppy
hpy = guppy.hpy()
except:
pass
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
return
s = time.time()
# Style
yield """
<style>
* { font-family: monospace }
table td, table th { text-align: right; padding: 0px 10px }
.connections td { white-space: nowrap }
.serving-False { opacity: 0.3 }
</style>
"""
# Memory
yield "rev%s | " % config.rev
yield "%s | " % main.file_server.ip_external_list
yield "Port: %s | " % main.file_server.port
yield "Network: %s | " % main.file_server.supported_ip_types
yield "Opened: %s | " % main.file_server.port_opened
yield "Crypt: %s, TLSv1.3: %s | " % (CryptConnection.manager.crypt_supported, CryptConnection.ssl.HAS_TLSv1_3)
yield "In: %.2fMB, Out: %.2fMB | " % (
float(main.file_server.bytes_recv) / 1024 / 1024,
float(main.file_server.bytes_sent) / 1024 / 1024
)
yield "Peerid: %s | " % main.file_server.peer_id
yield "Time: %.2fs | " % main.file_server.getTimecorrection()
yield "Blocks: %s" % Debug.num_block
try:
import psutil
process = psutil.Process(os.getpid())
mem = process.get_memory_info()[0] / float(2 ** 20)
yield "Mem: %.2fMB | " % mem
yield "Threads: %s | " % len(process.threads())
yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
yield "Files: %s | " % len(process.open_files())
yield "Sockets: %s | " % len(process.connections())
yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
except Exception:
pass
yield "<br>"
# Connections
yield "<b>Connections</b> (%s, total made: %s, in: %s, out: %s):<br>" % (
len(main.file_server.connections), main.file_server.last_connection_id, main.file_server.num_incoming, main.file_server.num_outgoing
)
yield "<table class='connections'><tr> <th>id</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>"
yield "<th>wait</th> <th>version</th> <th>time</th> <th>sites</th> </tr>"
for connection in main.file_server.connections:
if "cipher" in dir(connection.sock):
cipher = connection.sock.cipher()[0]
tls_version = connection.sock.version()
else:
cipher = connection.crypt
tls_version = ""
if "time" in connection.handshake and connection.last_ping_delay:
time_correction = connection.handshake["time"] - connection.handshake_time - connection.last_ping_delay
else:
time_correction = 0.0
yield self.formatTableRow([
("%3d", connection.id),
("%s", connection.type),
("%s:%s", (connection.ip, connection.port)),
("%s", connection.handshake.get("port_opened")),
("<span title='%s %s'>%s</span>", (cipher, tls_version, connection.crypt)),
("%6.3f", connection.last_ping_delay),
("%s", connection.incomplete_buff_recv),
("%s", connection.bad_actions),
("since", max(connection.last_send_time, connection.last_recv_time)),
("since", connection.start_time),
("%.3f", max(-1, connection.last_sent_time - connection.last_send_time)),
("%.3f", connection.cpu_time),
("%.0fk", connection.bytes_sent / 1024),
("%.0fk", connection.bytes_recv / 1024),
("<span title='Recv: %s'>%s</span>", (connection.last_cmd_recv, connection.last_cmd_sent)),
("%s", list(connection.waiting_requests.keys())),
("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))),
("%.2fs", time_correction),
("%s", connection.sites)
])
yield "</table>"
# Trackers
yield "<br><br><b>Trackers:</b><br>"
yield "<table class='trackers'><tr> <th>address</th> <th>request</th> <th>successive errors</th> <th>last_request</th></tr>"
from Site import SiteAnnouncer # importing at the top of the file breaks plugins
for tracker_address, tracker_stat in sorted(SiteAnnouncer.global_stats.items()):
yield self.formatTableRow([
("%s", tracker_address),
("%s", tracker_stat["num_request"]),
("%s", tracker_stat["num_error"]),
("%.0f min ago", min(999, (time.time() - tracker_stat["time_request"]) / 60))
])
yield "</table>"
if "AnnounceShare" in PluginManager.plugin_manager.plugin_names:
yield "<br><br><b>Shared trackers:</b><br>"
yield "<table class='trackers'><tr> <th>address</th> <th>added</th> <th>found</th> <th>latency</th> <th>successive errors</th> <th>last_success</th></tr>"
from AnnounceShare import AnnounceSharePlugin
for tracker_address, tracker_stat in sorted(AnnounceSharePlugin.tracker_storage.getTrackers().items()):
yield self.formatTableRow([
("%s", tracker_address),
("%.0f min ago", min(999, (time.time() - tracker_stat["time_added"]) / 60)),
("%.0f min ago", min(999, (time.time() - tracker_stat.get("time_found", 0)) / 60)),
("%.3fs", tracker_stat["latency"]),
("%s", tracker_stat["num_error"]),
("%.0f min ago", min(999, (time.time() - tracker_stat["time_success"]) / 60)),
])
yield "</table>"
# Tor hidden services
yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
for site_address, onion in list(main.file_server.tor_manager.site_onions.items()):
yield "- %-34s: %s<br>" % (site_address, onion)
# Db
yield "<br><br><b>Db</b>:<br>"
for db in Db.opened_dbs:
tables = [row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()]
table_rows = {}
for table in tables:
table_rows[table] = db.execute("SELECT COUNT(*) AS c FROM %s" % table).fetchone()["c"]
db_size = os.path.getsize(db.db_path) / 1024.0 / 1024.0
yield "- %.3fs: %s %.3fMB, table rows: %s<br>" % (
time.time() - db.last_query_time, db.db_path, db_size, json.dumps(table_rows, sort_keys=True)
)
# Sites
yield "<br><br><b>Sites</b>:"
yield "<table>"
yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>"
for site in list(self.server.sites.values()):
yield self.formatTableRow([
(
"""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
(site.address, site.address)
),
("%s", [peer.connection.id for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]),
("%s/%s/%s", (
len([peer for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]),
len(site.getConnectablePeers(100)),
len(site.peers)
)),
("%s (loaded: %s)", (
len(site.content_manager.contents),
len([key for key, val in dict(site.content_manager.contents).items() if val])
)),
("%.0fk", site.settings.get("bytes_sent", 0) / 1024),
("%.0fk", site.settings.get("bytes_recv", 0) / 1024),
], "serving-%s" % site.settings["serving"])
yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
for key, peer in list(site.peers.items()):
if peer.time_found:
time_found = int(time.time() - peer.time_found) / 60
else:
time_found = "--"
if peer.connection:
connection_id = peer.connection.id
else:
connection_id = None
if site.content_manager.has_optional_files:
yield "Optional files: %4s " % len(peer.hashfield)
time_added = (time.time() - peer.time_added) / (60 * 60 * 24)
yield "(#%4s, rep: %2s, err: %s, found: %.1fs min, add: %.1f day) %30s -<br>" % (connection_id, peer.reputation, peer.connection_error, time_found, time_added, key)
yield "<br></td></tr>"
yield "</table>"
# Big files
yield "<br><br><b>Big files</b>:<br>"
for site in list(self.server.sites.values()):
if not site.settings.get("has_bigfile"):
continue
bigfiles = {}
yield """<a href="#" onclick='document.getElementById("bigfiles_%s").style.display="initial"; return false'>%s</a><br>""" % (site.address, site.address)
for peer in list(site.peers.values()):
if not peer.time_piecefields_updated:
continue
for sha512, piecefield in peer.piecefields.items():
if sha512 not in bigfiles:
bigfiles[sha512] = []
bigfiles[sha512].append(peer)
yield "<div id='bigfiles_%s' style='display: none'>" % site.address
for sha512, peers in bigfiles.items():
yield "<br> - " + sha512 + " (hash id: %s)<br>" % site.content_manager.hashfield.getHashId(sha512)
yield "<table>"
for peer in peers:
yield "<tr><td>" + peer.key + "</td><td>" + peer.piecefields[sha512].tostring() + "</td></tr>"
yield "</table>"
yield "</div>"
# Cmd stats
yield "<div style='float: left'>"
yield "<br><br><b>Sent commands</b>:<br>"
yield "<table>"
for stat_key, stat in sorted(main.file_server.stat_sent.items(), key=lambda i: i[1]["bytes"], reverse=True):
yield "<tr><td>%s</td><td style='white-space: nowrap'>x %s =</td><td>%.0fkB</td></tr>" % (stat_key, stat["num"], stat["bytes"] / 1024)
yield "</table>"
yield "</div>"
yield "<div style='float: left; margin-left: 20%; max-width: 50%'>"
yield "<br><br><b>Received commands</b>:<br>"
yield "<table>"
for stat_key, stat in sorted(main.file_server.stat_recv.items(), key=lambda i: i[1]["bytes"], reverse=True):
yield "<tr><td>%s</td><td style='white-space: nowrap'>x %s =</td><td>%.0fkB</td></tr>" % (stat_key, stat["num"], stat["bytes"] / 1024)
yield "</table>"
yield "</div>"
yield "<div style='clear: both'></div>"
# No more if not in debug mode
if not config.debug:
return
# Object types
obj_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type not in obj_count:
obj_count[obj_type] = [0, 0]
obj_count[obj_type][0] += 1 # Count
obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
len(obj_count),
sum([stat[0] for stat in list(obj_count.values())]),
sum([stat[1] for stat in list(obj_count.values())])
)
for obj, stat in sorted(list(obj_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count
yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, html.escape(obj))
# Classes
class_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type != "<type 'instance'>":
continue
class_name = obj.__class__.__name__
if class_name not in class_count:
class_count[class_name] = [0, 0]
class_count[class_name][0] += 1 # Count
class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
len(class_count),
sum([stat[0] for stat in list(class_count.values())]),
sum([stat[1] for stat in list(class_count.values())])
)
for obj, stat in sorted(list(class_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count
yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, html.escape(obj))
from greenlet import greenlet
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
yield "<br>Greenlets (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Worker import Worker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
yield "<br>Workers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Connection import Connection
objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
yield "<br>Connections (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from socket import socket
objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
yield "<br>Sockets (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from msgpack import Unpacker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Site.Site import Site
objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
yield "<br>Sites (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
yield "<br>Loggers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj.name)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
yield "<br>UiRequests (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Peer import Peer
objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
yield "<br>Peers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
objs = [(key, val) for key, val in sys.modules.items() if val is not None]
objs.sort()
yield "<br>Modules (%s):<br>" % len(objs)
for module_name, module in objs:
yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, html.escape(repr(module)))
gc.collect() # Implicit grabage collection
yield "Done in %.1f" % (time.time() - s)
@helper.encodeResponse
def actionDumpobj(self):
import gc
import sys
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
return
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
return
class_filter = self.get.get("class")
yield """
<style>
* { font-family: monospace; white-space: pre }
table * { text-align: right; padding: 0px 10px }
</style>
"""
objs = gc.get_objects()
for obj in objs:
obj_type = str(type(obj))
if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter:
continue
yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, html.escape(str(obj)))
for attr in dir(obj):
yield "- %s: %s<br>" % (attr, html.escape(str(getattr(obj, attr))))
yield "<br>"
gc.collect() # Implicit grabage collection
@helper.encodeResponse
def actionListobj(self):
import gc
import sys
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
return
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
return
type_filter = self.get.get("type")
yield """
<style>
* { font-family: monospace; white-space: pre }
table * { text-align: right; padding: 0px 10px }
</style>
"""
yield "Listing all %s objects in memory...<br>" % html.escape(type_filter)
ref_count = {}
objs = gc.get_objects()
for obj in objs:
obj_type = str(type(obj))
if obj_type != type_filter:
continue
refs = [
ref for ref in gc.get_referrers(obj)
if hasattr(ref, "__class__") and
ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]
]
if not refs:
continue
try:
yield "%.1fkb <span title=\"%s\">%s</span>... " % (
float(sys.getsizeof(obj)) / 1024, html.escape(str(obj)), html.escape(str(obj)[0:100].ljust(100))
)
except:
continue
for ref in refs:
yield " ["
if "object at" in str(ref) or len(str(ref)) > 100:
yield str(ref.__class__.__name__)
else:
yield str(ref.__class__.__name__) + ":" + html.escape(str(ref))
yield "] "
ref_type = ref.__class__.__name__
if ref_type not in ref_count:
ref_count[ref_type] = [0, 0]
ref_count[ref_type][0] += 1 # Count
ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br>"
yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in list(ref_count.values())]))
for obj, stat in sorted(list(ref_count.items()), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count
yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], html.escape(str(obj)))
gc.collect() # Implicit grabage collection
@helper.encodeResponse
def actionGcCollect(self):
import gc
self.sendHeader()
yield str(gc.collect())