rev259, Protection against connection flood, Fix site size limit error dialog, Convert ConnectionServer and ContentManager to PEP8 format
This commit is contained in:
parent
f63b711972
commit
dc791a31ab
6 changed files with 301 additions and 339 deletions
|
@ -4,7 +4,7 @@ import ConfigParser
|
||||||
class Config(object):
|
class Config(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.version = "0.3.1"
|
self.version = "0.3.1"
|
||||||
self.rev = 247
|
self.rev = 259
|
||||||
self.parser = self.createArguments()
|
self.parser = self.createArguments()
|
||||||
argv = sys.argv[:] # Copy command line arguments
|
argv = sys.argv[:] # Copy command line arguments
|
||||||
argv = self.parseConfig(argv) # Add arguments from config file
|
argv = self.parseConfig(argv) # Add arguments from config file
|
||||||
|
|
|
@ -1,8 +1,14 @@
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
import string
|
||||||
|
import time
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import gevent
|
||||||
|
import msgpack
|
||||||
from gevent.server import StreamServer
|
from gevent.server import StreamServer
|
||||||
from gevent.pool import Pool
|
from gevent.pool import Pool
|
||||||
import socket, os, logging, random, string, time, sys
|
|
||||||
import gevent, msgpack
|
|
||||||
import cStringIO as StringIO
|
|
||||||
from Debug import Debug
|
from Debug import Debug
|
||||||
from Connection import Connection
|
from Connection import Connection
|
||||||
from Config import config
|
from Config import config
|
||||||
|
@ -10,250 +16,163 @@ from Crypt import CryptConnection
|
||||||
|
|
||||||
|
|
||||||
class ConnectionServer:
|
class ConnectionServer:
|
||||||
def __init__(self, ip=None, port=None, request_handler=None):
|
def __init__(self, ip=None, port=None, request_handler=None):
|
||||||
self.ip = ip
|
self.ip = ip
|
||||||
self.port = port
|
self.port = port
|
||||||
self.last_connection_id = 1 # Connection id incrementer
|
self.last_connection_id = 1 # Connection id incrementer
|
||||||
self.log = logging.getLogger("ConnServer")
|
self.log = logging.getLogger("ConnServer")
|
||||||
self.port_opened = None
|
self.port_opened = None
|
||||||
|
|
||||||
self.connections = [] # Connections
|
self.connections = [] # Connections
|
||||||
self.ips = {} # Connection by ip
|
self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood
|
||||||
self.peer_ids = {} # Connections by peer_ids
|
self.ips = {} # Connection by ip
|
||||||
|
self.peer_ids = {} # Connections by peer_ids
|
||||||
|
|
||||||
self.running = True
|
self.running = True
|
||||||
self.thread_checker = gevent.spawn(self.checkConnections)
|
self.thread_checker = gevent.spawn(self.checkConnections)
|
||||||
|
|
||||||
self.bytes_recv = 0
|
self.bytes_recv = 0
|
||||||
self.bytes_sent = 0
|
self.bytes_sent = 0
|
||||||
|
|
||||||
self.peer_id = "-ZN0"+config.version.replace(".", "")+"-"+''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12)) # Bittorrent style peerid
|
# Bittorrent style peerid
|
||||||
|
self.peer_id = "-ZN0%s-%s" % (
|
||||||
|
config.version.replace(".", ""),
|
||||||
|
''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
|
||||||
|
)
|
||||||
|
|
||||||
# Check msgpack version
|
# Check msgpack version
|
||||||
if msgpack.version[0] == 0 and msgpack.version[1] < 4:
|
if msgpack.version[0] == 0 and msgpack.version[1] < 4:
|
||||||
self.log.error("Error: Too old msgpack version: %s (>0.4.0 required), please update using `sudo pip install msgpack-python --upgrade`" % str(msgpack.version))
|
self.log.error(
|
||||||
import sys
|
"Error: Too old msgpack version: %s (>0.4.0 required), please update using `sudo pip install msgpack-python --upgrade`" %
|
||||||
sys.exit(0)
|
str(msgpack.version)
|
||||||
|
)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
if port: # Listen server on a port
|
if port: # Listen server on a port
|
||||||
self.pool = Pool(1000) # do not accept more than 1000 connections
|
self.pool = Pool(1000) # do not accept more than 1000 connections
|
||||||
self.stream_server = StreamServer((ip.replace("*", ""), port), self.handleIncomingConnection, spawn=self.pool, backlog=100)
|
self.stream_server = StreamServer((ip.replace("*", ""), port), self.handleIncomingConnection, spawn=self.pool, backlog=100)
|
||||||
if request_handler: self.handleRequest = request_handler
|
if request_handler:
|
||||||
|
self.handleRequest = request_handler
|
||||||
|
|
||||||
CryptConnection.manager.loadCerts()
|
CryptConnection.manager.loadCerts()
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self.running = True
|
||||||
|
self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % (
|
||||||
|
self.ip, self.port,
|
||||||
|
".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
self.stream_server.serve_forever() # Start normal connection server
|
||||||
|
except Exception, err:
|
||||||
|
self.log.info("StreamServer bind error, must be running already: %s" % err)
|
||||||
|
|
||||||
def start(self):
|
def stop(self):
|
||||||
self.running = True
|
self.running = False
|
||||||
self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % (self.ip, self.port, ".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported ) )
|
self.stream_server.stop()
|
||||||
try:
|
|
||||||
self.stream_server.serve_forever() # Start normal connection server
|
|
||||||
except Exception, err:
|
|
||||||
self.log.info("StreamServer bind error, must be running already: %s" % err)
|
|
||||||
|
|
||||||
|
def handleIncomingConnection(self, sock, addr):
|
||||||
|
ip, port = addr
|
||||||
|
|
||||||
def stop(self):
|
# Connection flood protection
|
||||||
self.running = False
|
if ip in self.ip_incoming:
|
||||||
self.stream_server.stop()
|
self.ip_incoming[ip] += 1
|
||||||
|
if self.ip_incoming[ip] > 3: # Allow 3 in 1 minute from same ip
|
||||||
|
self.log.debug("Connection flood detected from %s" % ip)
|
||||||
|
time.sleep(30)
|
||||||
|
sock.close()
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
self.ip_incoming[ip] = 0
|
||||||
|
|
||||||
|
connection = Connection(self, ip, port, sock)
|
||||||
|
self.connections.append(connection)
|
||||||
|
self.ips[ip] = connection
|
||||||
|
connection.handleIncomingConnection(sock)
|
||||||
|
|
||||||
def handleIncomingConnection(self, sock, addr):
|
def getConnection(self, ip=None, port=None, peer_id=None, create=True):
|
||||||
ip, port = addr
|
if peer_id and peer_id in self.peer_ids: # Find connection by peer id
|
||||||
connection = Connection(self, ip, port, sock)
|
connection = self.peer_ids.get(peer_id)
|
||||||
self.connections.append(connection)
|
if not connection.connected and create:
|
||||||
self.ips[ip] = connection
|
succ = connection.event_connected.get() # Wait for connection
|
||||||
connection.handleIncomingConnection(sock)
|
if not succ:
|
||||||
|
raise Exception("Connection event return error")
|
||||||
|
return connection
|
||||||
|
# Find connection by ip
|
||||||
|
if ip in self.ips:
|
||||||
|
connection = self.ips[ip]
|
||||||
|
if not connection.connected and create:
|
||||||
|
succ = connection.event_connected.get() # Wait for connection
|
||||||
|
if not succ:
|
||||||
|
raise Exception("Connection event return error")
|
||||||
|
return connection
|
||||||
|
# Recover from connection pool
|
||||||
|
for connection in self.connections:
|
||||||
|
if connection.ip == ip:
|
||||||
|
if not connection.connected and create:
|
||||||
|
succ = connection.event_connected.get() # Wait for connection
|
||||||
|
if not succ:
|
||||||
|
raise Exception("Connection event return error")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
# No connection found
|
||||||
|
if create: # Allow to create new connection if not found
|
||||||
|
if port == 0:
|
||||||
|
raise Exception("This peer is not connectable")
|
||||||
|
try:
|
||||||
|
connection = Connection(self, ip, port)
|
||||||
|
self.ips[ip] = connection
|
||||||
|
self.connections.append(connection)
|
||||||
|
succ = connection.connect()
|
||||||
|
if not succ:
|
||||||
|
connection.close()
|
||||||
|
raise Exception("Connection event return error")
|
||||||
|
|
||||||
|
except Exception, err:
|
||||||
|
self.log.debug("%s Connect error: %s" % (ip, Debug.formatException(err)))
|
||||||
|
connection.close()
|
||||||
|
raise err
|
||||||
|
return connection
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
def getConnection(self, ip=None, port=None, peer_id=None, create=True):
|
def removeConnection(self, connection):
|
||||||
if peer_id and peer_id in self.peer_ids: # Find connection by peer id
|
self.log.debug("Removing %s..." % connection)
|
||||||
connection = self.peer_ids.get(peer_id)
|
if self.ips.get(connection.ip) == connection: # Delete if same as in registry
|
||||||
if not connection.connected and create:
|
del self.ips[connection.ip]
|
||||||
succ = connection.event_connected.get() # Wait for connection
|
if connection.peer_id and self.peer_ids.get(connection.peer_id) == connection: # Delete if same as in registry
|
||||||
if not succ: raise Exception("Connection event return error")
|
del self.peer_ids[connection.peer_id]
|
||||||
return connection
|
if connection in self.connections:
|
||||||
# Find connection by ip
|
self.connections.remove(connection)
|
||||||
if ip in self.ips:
|
|
||||||
connection = self.ips[ip]
|
|
||||||
if not connection.connected and create:
|
|
||||||
succ = connection.event_connected.get() # Wait for connection
|
|
||||||
if not succ: raise Exception("Connection event return error")
|
|
||||||
return connection
|
|
||||||
# Recover from connection pool
|
|
||||||
for connection in self.connections:
|
|
||||||
if connection.ip == ip:
|
|
||||||
if not connection.connected and create:
|
|
||||||
succ = connection.event_connected.get() # Wait for connection
|
|
||||||
if not succ: raise Exception("Connection event return error")
|
|
||||||
return connection
|
|
||||||
|
|
||||||
# No connection found
|
def checkConnections(self):
|
||||||
if create: # Allow to create new connection if not found
|
while self.running:
|
||||||
if port == 0:
|
time.sleep(60) # Sleep 1 min
|
||||||
raise Exception("This peer is not connectable")
|
self.ip_incoming = {}
|
||||||
try:
|
for connection in self.connections[:]: # Make a copy
|
||||||
connection = Connection(self, ip, port)
|
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
||||||
self.ips[ip] = connection
|
|
||||||
self.connections.append(connection)
|
|
||||||
succ = connection.connect()
|
|
||||||
if not succ:
|
|
||||||
connection.close()
|
|
||||||
raise Exception("Connection event return error")
|
|
||||||
|
|
||||||
except Exception, err:
|
if connection.unpacker and idle > 30: # Delete the unpacker if not needed
|
||||||
self.log.debug("%s Connect error: %s" % (ip, Debug.formatException(err)))
|
del connection.unpacker
|
||||||
connection.close()
|
connection.unpacker = None
|
||||||
raise err
|
connection.log("Unpacker deleted")
|
||||||
return connection
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
if idle > 60 * 60: # Wake up after 1h
|
||||||
|
connection.log("[Cleanup] After wakeup, idle: %s" % idle)
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
elif idle > 20 * 60 and connection.last_send_time < time.time() - 10: # Idle more than 20 min and we not send request in last 10 sec
|
||||||
|
if not connection.ping(): # send ping request
|
||||||
|
connection.close()
|
||||||
|
|
||||||
def removeConnection(self, connection):
|
elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle
|
||||||
self.log.debug("Removing %s..." % connection)
|
connection.log("[Cleanup] Connection buff stalled")
|
||||||
if self.ips.get(connection.ip) == connection: # Delete if same as in registry
|
connection.close()
|
||||||
del self.ips[connection.ip]
|
|
||||||
if connection.peer_id and self.peer_ids.get(connection.peer_id) == connection: # Delete if same as in registry
|
|
||||||
del self.peer_ids[connection.peer_id]
|
|
||||||
if connection in self.connections:
|
|
||||||
self.connections.remove(connection)
|
|
||||||
|
|
||||||
|
elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec
|
||||||
|
connection.log("[Cleanup] Command %s timeout: %s" % (connection.last_cmd, time.time() - connection.last_send_time))
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
elif idle > 60 and connection.protocol == "?": # No connection after 1 min
|
||||||
def checkConnections(self):
|
connection.log("[Cleanup] Connect timeout: %s" % idle)
|
||||||
while self.running:
|
connection.close()
|
||||||
time.sleep(60) # Sleep 1 min
|
|
||||||
for connection in self.connections[:]: # Make a copy
|
|
||||||
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
|
||||||
|
|
||||||
if connection.unpacker and idle > 30: # Delete the unpacker if not needed
|
|
||||||
del connection.unpacker
|
|
||||||
connection.unpacker = None
|
|
||||||
connection.log("Unpacker deleted")
|
|
||||||
|
|
||||||
if idle > 60*60: # Wake up after 1h
|
|
||||||
connection.log("[Cleanup] After wakeup, idle: %s" % idle)
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
elif idle > 20*60 and connection.last_send_time < time.time()-10: # Idle more than 20 min and we not send request in last 10 sec
|
|
||||||
if not connection.ping(): # send ping request
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle
|
|
||||||
connection.log("[Cleanup] Connection buff stalled")
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec
|
|
||||||
connection.log("[Cleanup] Command %s timeout: %s" % (connection.last_cmd, time.time() - connection.last_send_time))
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
elif idle > 60 and connection.protocol == "?": # No connection after 1 min
|
|
||||||
connection.log("[Cleanup] Connect timeout: %s" % idle)
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
# -- TESTING --
|
|
||||||
|
|
||||||
def testCreateServer():
|
|
||||||
global server
|
|
||||||
server = ConnectionServer("127.0.0.1", 1234, testRequestHandler)
|
|
||||||
server.start()
|
|
||||||
|
|
||||||
|
|
||||||
def testRequestHandler(connection, req):
|
|
||||||
print req
|
|
||||||
if req["cmd"] == "Bigdata":
|
|
||||||
connection.send({"res": "HelloWorld"*1024})
|
|
||||||
else:
|
|
||||||
connection.send({"res": "pong"})
|
|
||||||
|
|
||||||
|
|
||||||
def testClient(num):
|
|
||||||
time.sleep(1)
|
|
||||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
||||||
s.connect(("localhost", 1234))
|
|
||||||
for i in range(10):
|
|
||||||
print "[C%s] send..." % num
|
|
||||||
s.sendall(msgpack.packb({"cmd": "[C] Ping"}))
|
|
||||||
print "[C%s] recv..." % num
|
|
||||||
print "[C%s] %s" % (num, repr(s.recv(1024)))
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
|
|
||||||
def testSlowClient(num):
|
|
||||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
||||||
s.connect(("localhost", 1234))
|
|
||||||
for i in range(1):
|
|
||||||
print "[C%s] send..." % num
|
|
||||||
s.sendall(msgpack.packb({"cmd": "Bigdata"}))
|
|
||||||
print "[C%s] recv..." % num
|
|
||||||
gevent.spawn_later(1, lambda s: s.send(msgpack.packb({"cmd": "[Z] Ping"})), s)
|
|
||||||
while 1:
|
|
||||||
data = s.recv(1000)
|
|
||||||
if not data: break
|
|
||||||
print "[C%s] %s" % (num, data)
|
|
||||||
time.sleep(1)
|
|
||||||
#s.sendall(msgpack.packb({"cmd": "[C] Ping"}))
|
|
||||||
|
|
||||||
|
|
||||||
def testZmqClient(num):
|
|
||||||
import zmq.green as zmq
|
|
||||||
c = zmq.Context(1)
|
|
||||||
for i in range(10):
|
|
||||||
s = c.socket(zmq.REQ)
|
|
||||||
s.connect('tcp://127.0.0.1:1234')
|
|
||||||
print "[Z%s] send..." % num
|
|
||||||
s.send(msgpack.packb({"cmd": "[Z] Ping %s" % i}))
|
|
||||||
print "[Z%s] recv..." % num
|
|
||||||
print "[Z%s] %s" % (num, s.recv(1024))
|
|
||||||
s.close()
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
|
|
||||||
def testZmqSlowClient(num):
|
|
||||||
import zmq.green as zmq
|
|
||||||
c = zmq.Context(1)
|
|
||||||
s = c.socket(zmq.REQ)
|
|
||||||
for i in range(1):
|
|
||||||
s.connect('tcp://127.0.0.1:1234')
|
|
||||||
print "[Z%s] send..." % num
|
|
||||||
s.send(msgpack.packb({"cmd": "Bigdata"}))
|
|
||||||
print "[Z%s] recv..." % num
|
|
||||||
#gevent.spawn_later(1, lambda s: s.send(msgpack.packb({"cmd": "[Z] Ping"})), s)
|
|
||||||
while 1:
|
|
||||||
data = s.recv(1024*1024)
|
|
||||||
if not data: break
|
|
||||||
print "[Z%s] %s" % (num, data)
|
|
||||||
time.sleep(1)
|
|
||||||
s.send(msgpack.packb({"cmd": "[Z] Ping"}))
|
|
||||||
|
|
||||||
|
|
||||||
def testConnection():
|
|
||||||
global server
|
|
||||||
connection = server.getConnection("127.0.0.1", 1234)
|
|
||||||
connection.send({"res": "Sending: Hello!"})
|
|
||||||
print connection
|
|
||||||
|
|
||||||
|
|
||||||
def greenletsNum():
|
|
||||||
from greenlet import greenlet
|
|
||||||
import gc
|
|
||||||
while 1:
|
|
||||||
print len([ob for ob in gc.get_objects() if isinstance(ob, greenlet)])
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
from gevent import monkey; monkey.patch_all(thread=False)
|
|
||||||
import sys, time
|
|
||||||
logging.getLogger().setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
gevent.spawn(testZmqClient, 1)
|
|
||||||
gevent.spawn(greenletsNum)
|
|
||||||
#gevent.spawn(testClient, 1)
|
|
||||||
#gevent.spawn_later(1, testConnection)
|
|
||||||
print "Running server..."
|
|
||||||
server = None
|
|
||||||
testCreateServer()
|
|
||||||
|
|
||||||
|
|
|
@ -1,20 +1,29 @@
|
||||||
import json, time, re, os, gevent, copy
|
import json
|
||||||
|
import time
|
||||||
|
import re
|
||||||
|
import os
|
||||||
|
import copy
|
||||||
|
|
||||||
|
import gevent
|
||||||
|
|
||||||
from Debug import Debug
|
from Debug import Debug
|
||||||
from Crypt import CryptHash
|
from Crypt import CryptHash
|
||||||
from Config import config
|
from Config import config
|
||||||
|
|
||||||
|
|
||||||
class ContentManager(object):
|
class ContentManager(object):
|
||||||
|
|
||||||
def __init__(self, site):
|
def __init__(self, site):
|
||||||
self.site = site
|
self.site = site
|
||||||
self.log = self.site.log
|
self.log = self.site.log
|
||||||
self.contents = {} # Known content.json (without files and includes)
|
self.contents = {} # Known content.json (without files and includes)
|
||||||
self.loadContent(add_bad_files=False)
|
self.loadContent(add_bad_files=False)
|
||||||
self.site.settings["size"] = self.getTotalSize()
|
self.site.settings["size"] = self.getTotalSize()
|
||||||
|
|
||||||
# Load content.json to self.content
|
# Load content.json to self.content
|
||||||
# Return: Changed files ["index.html", "data/messages.json"]
|
# Return: Changed files ["index.html", "data/messages.json"]
|
||||||
def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True):
|
def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True):
|
||||||
content_inner_path = content_inner_path.strip("/") # Remove / from begning
|
content_inner_path = content_inner_path.strip("/") # Remove / from begning
|
||||||
old_content = self.contents.get(content_inner_path)
|
old_content = self.contents.get(content_inner_path)
|
||||||
content_path = self.site.storage.getPath(content_inner_path)
|
content_path = self.site.storage.getPath(content_inner_path)
|
||||||
content_path_dir = self.toDir(self.site.storage.getPath(content_inner_path))
|
content_path_dir = self.toDir(self.site.storage.getPath(content_inner_path))
|
||||||
|
@ -28,7 +37,7 @@ class ContentManager(object):
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
self.log.error("Content.json not exist: %s" % content_path)
|
self.log.error("Content.json not exist: %s" % content_path)
|
||||||
return False # Content.json not exist
|
return False # Content.json not exist
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Get the files where the sha512 changed
|
# Get the files where the sha512 changed
|
||||||
|
@ -36,40 +45,44 @@ class ContentManager(object):
|
||||||
for relative_path, info in new_content.get("files", {}).items():
|
for relative_path, info in new_content.get("files", {}).items():
|
||||||
if "sha512" in info:
|
if "sha512" in info:
|
||||||
hash_type = "sha512"
|
hash_type = "sha512"
|
||||||
else: # Backward compatiblity
|
else: # Backward compatiblity
|
||||||
hash_type = "sha1"
|
hash_type = "sha1"
|
||||||
|
|
||||||
new_hash = info[hash_type]
|
new_hash = info[hash_type]
|
||||||
if old_content and old_content["files"].get(relative_path): # We have the file in the old content
|
if old_content and old_content["files"].get(relative_path): # We have the file in the old content
|
||||||
old_hash = old_content["files"][relative_path].get(hash_type)
|
old_hash = old_content["files"][relative_path].get(hash_type)
|
||||||
else: # The file is not in the old content
|
else: # The file is not in the old content
|
||||||
old_hash = None
|
old_hash = None
|
||||||
if old_hash != new_hash: changed.append(content_dir+relative_path)
|
if old_hash != new_hash:
|
||||||
|
changed.append(content_dir + relative_path)
|
||||||
|
|
||||||
# Load includes
|
# Load includes
|
||||||
if load_includes and "includes" in new_content:
|
if load_includes and "includes" in new_content:
|
||||||
for relative_path, info in new_content["includes"].items():
|
for relative_path, info in new_content["includes"].items():
|
||||||
include_inner_path = content_dir+relative_path
|
include_inner_path = content_dir + relative_path
|
||||||
if self.site.storage.isFile(include_inner_path): # Content.json exists, load it
|
if self.site.storage.isFile(include_inner_path): # Content.json exists, load it
|
||||||
success = self.loadContent(include_inner_path, add_bad_files=add_bad_files)
|
success = self.loadContent(include_inner_path, add_bad_files=add_bad_files)
|
||||||
if success: changed += success # Add changed files
|
if success:
|
||||||
else: # Content.json not exist, add to changed files
|
changed += success # Add changed files
|
||||||
|
else: # Content.json not exist, add to changed files
|
||||||
self.log.debug("Missing include: %s" % include_inner_path)
|
self.log.debug("Missing include: %s" % include_inner_path)
|
||||||
changed += [include_inner_path]
|
changed += [include_inner_path]
|
||||||
|
|
||||||
# Load blind user includes (all subdir)
|
# Load blind user includes (all subdir)
|
||||||
if load_includes and "user_contents" in new_content:
|
if load_includes and "user_contents" in new_content:
|
||||||
for relative_dir in os.listdir(content_path_dir):
|
for relative_dir in os.listdir(content_path_dir):
|
||||||
include_inner_path = content_dir+relative_dir+"/content.json"
|
include_inner_path = content_dir + relative_dir + "/content.json"
|
||||||
if not self.site.storage.isFile(include_inner_path): continue # Content.json not exist
|
if not self.site.storage.isFile(include_inner_path):
|
||||||
|
continue # Content.json not exist
|
||||||
success = self.loadContent(include_inner_path, add_bad_files=add_bad_files, load_includes=False)
|
success = self.loadContent(include_inner_path, add_bad_files=add_bad_files, load_includes=False)
|
||||||
if success: changed += success # Add changed files
|
if success:
|
||||||
|
changed += success # Add changed files
|
||||||
|
|
||||||
# Update the content
|
# Update the content
|
||||||
self.contents[content_inner_path] = new_content
|
self.contents[content_inner_path] = new_content
|
||||||
except Exception, err:
|
except Exception, err:
|
||||||
self.log.error("Content.json parse error: %s" % Debug.formatException(err))
|
self.log.error("Content.json parse error: %s" % Debug.formatException(err))
|
||||||
return False # Content.json parse error
|
return False # Content.json parse error
|
||||||
|
|
||||||
# Add changed files to bad files
|
# Add changed files to bad files
|
||||||
if add_bad_files:
|
if add_bad_files:
|
||||||
|
@ -77,7 +90,8 @@ class ContentManager(object):
|
||||||
self.site.bad_files[inner_path] = True
|
self.site.bad_files[inner_path] = True
|
||||||
|
|
||||||
if new_content["modified"] > self.site.settings.get("modified", 0):
|
if new_content["modified"] > self.site.settings.get("modified", 0):
|
||||||
self.site.settings["modified"] = min(time.time()+60*10, new_content["modified"]) # Dont store modifications in the far future (more than 10 minute)
|
# Dont store modifications in the far future (more than 10 minute)
|
||||||
|
self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"])
|
||||||
|
|
||||||
return changed
|
return changed
|
||||||
|
|
||||||
|
@ -86,8 +100,9 @@ class ContentManager(object):
|
||||||
def getTotalSize(self, ignore=None):
|
def getTotalSize(self, ignore=None):
|
||||||
total_size = 0
|
total_size = 0
|
||||||
for inner_path, content in self.contents.iteritems():
|
for inner_path, content in self.contents.iteritems():
|
||||||
if inner_path == ignore: continue
|
if inner_path == ignore:
|
||||||
total_size += self.site.storage.getSize(inner_path) # Size of content.json
|
continue
|
||||||
|
total_size += self.site.storage.getSize(inner_path) # Size of content.json
|
||||||
for file, info in content.get("files", {}).iteritems():
|
for file, info in content.get("files", {}).iteritems():
|
||||||
total_size += info["size"]
|
total_size += info["size"]
|
||||||
return total_size
|
return total_size
|
||||||
|
@ -95,12 +110,12 @@ class ContentManager(object):
|
||||||
# Find the file info line from self.contents
|
# Find the file info line from self.contents
|
||||||
# Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"}
|
# Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"}
|
||||||
def getFileInfo(self, inner_path):
|
def getFileInfo(self, inner_path):
|
||||||
dirs = inner_path.split("/") # Parent dirs of content.json
|
dirs = inner_path.split("/") # Parent dirs of content.json
|
||||||
inner_path_parts = [dirs.pop()] # Filename relative to content.json
|
inner_path_parts = [dirs.pop()] # Filename relative to content.json
|
||||||
while True:
|
while True:
|
||||||
content_inner_path = "%s/content.json" % "/".join(dirs)
|
content_inner_path = "%s/content.json" % "/".join(dirs)
|
||||||
content = self.contents.get(content_inner_path.strip("/"))
|
content = self.contents.get(content_inner_path.strip("/"))
|
||||||
if content and "files" in content: # Check if content.json exists
|
if content and "files" in content: # Check if content.json exists
|
||||||
back = content["files"].get("/".join(inner_path_parts))
|
back = content["files"].get("/".join(inner_path_parts))
|
||||||
if back:
|
if back:
|
||||||
back["content_inner_path"] = content_inner_path
|
back["content_inner_path"] = content_inner_path
|
||||||
|
@ -124,13 +139,14 @@ class ContentManager(object):
|
||||||
# Get rules for the file
|
# Get rules for the file
|
||||||
# Return: The rules for the file or False if not allowed
|
# Return: The rules for the file or False if not allowed
|
||||||
def getRules(self, inner_path, content=None):
|
def getRules(self, inner_path, content=None):
|
||||||
if not inner_path.endswith("content.json"): # Find the files content.json first
|
if not inner_path.endswith("content.json"): # Find the files content.json first
|
||||||
file_info = self.getFileInfo(inner_path)
|
file_info = self.getFileInfo(inner_path)
|
||||||
if not file_info: return False # File not found
|
if not file_info:
|
||||||
|
return False # File not found
|
||||||
inner_path = file_info["content_inner_path"]
|
inner_path = file_info["content_inner_path"]
|
||||||
dirs = inner_path.split("/") # Parent dirs of content.json
|
dirs = inner_path.split("/") # Parent dirs of content.json
|
||||||
inner_path_parts = [dirs.pop()] # Filename relative to content.json
|
inner_path_parts = [dirs.pop()] # Filename relative to content.json
|
||||||
inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir
|
inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir
|
||||||
while True:
|
while True:
|
||||||
content_inner_path = "%s/content.json" % "/".join(dirs)
|
content_inner_path = "%s/content.json" % "/".join(dirs)
|
||||||
parent_content = self.contents.get(content_inner_path.strip("/"))
|
parent_content = self.contents.get(content_inner_path.strip("/"))
|
||||||
|
@ -138,23 +154,23 @@ class ContentManager(object):
|
||||||
return parent_content["includes"].get("/".join(inner_path_parts))
|
return parent_content["includes"].get("/".join(inner_path_parts))
|
||||||
elif parent_content and "user_contents" in parent_content:
|
elif parent_content and "user_contents" in parent_content:
|
||||||
return self.getUserContentRules(parent_content, inner_path, content)
|
return self.getUserContentRules(parent_content, inner_path, content)
|
||||||
else: # No inner path in this dir, lets try the parent dir
|
else: # No inner path in this dir, lets try the parent dir
|
||||||
if dirs:
|
if dirs:
|
||||||
inner_path_parts.insert(0, dirs.pop())
|
inner_path_parts.insert(0, dirs.pop())
|
||||||
else: # No more parent dirs
|
else: # No more parent dirs
|
||||||
break
|
break
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
# Get rules for a user file
|
# Get rules for a user file
|
||||||
# Return: The rules of the file or False if not allowed
|
# Return: The rules of the file or False if not allowed
|
||||||
def getUserContentRules(self, parent_content, inner_path, content):
|
def getUserContentRules(self, parent_content, inner_path, content):
|
||||||
user_contents = parent_content["user_contents"]
|
user_contents = parent_content["user_contents"]
|
||||||
user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory
|
user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not content: content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
|
if not content:
|
||||||
|
content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
|
||||||
except (Exception, ): # Content.json not exist
|
except (Exception, ): # Content.json not exist
|
||||||
return {"signers": [user_address], "user_address": user_address} # Return information that we know for sure
|
return {"signers": [user_address], "user_address": user_address} # Return information that we know for sure
|
||||||
|
|
||||||
|
@ -162,15 +178,16 @@ class ContentManager(object):
|
||||||
content["cert_auth_type"] = "unknown"
|
content["cert_auth_type"] = "unknown"
|
||||||
content["cert_user_name"] = "unknown@unknown"
|
content["cert_user_name"] = "unknown@unknown"
|
||||||
"""
|
"""
|
||||||
user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit
|
user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit
|
||||||
|
|
||||||
rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username
|
rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username
|
||||||
if rules == False:
|
if rules is False:
|
||||||
return False # User banned
|
return False # User banned
|
||||||
if "signers" in rules:
|
if "signers" in rules:
|
||||||
rules["signers"] = rules["signers"][:] # Make copy of the signers
|
rules["signers"] = rules["signers"][:] # Make copy of the signers
|
||||||
for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
|
for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
|
||||||
if not re.match(permission_pattern, user_urn): continue # Rule is not valid for user
|
if not re.match(permission_pattern, user_urn):
|
||||||
|
continue # Rule is not valid for user
|
||||||
# Update rules if its better than current recorded ones
|
# Update rules if its better than current recorded ones
|
||||||
for key, val in permission_rules.iteritems():
|
for key, val in permission_rules.iteritems():
|
||||||
if key not in rules:
|
if key not in rules:
|
||||||
|
@ -182,13 +199,15 @@ class ContentManager(object):
|
||||||
if val > rules[key]:
|
if val > rules[key]:
|
||||||
rules[key] = val
|
rules[key] = val
|
||||||
elif hasattr(val, "startswith"): # String, update if longer
|
elif hasattr(val, "startswith"): # String, update if longer
|
||||||
if len(val) > len(rules[key]): rules[key] = val
|
if len(val) > len(rules[key]):
|
||||||
|
rules[key] = val
|
||||||
elif type(val) is list: # List, append
|
elif type(val) is list: # List, append
|
||||||
rules[key] += val
|
rules[key] += val
|
||||||
|
|
||||||
rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers
|
rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers
|
||||||
if "signers" not in rules: rules["signers"] = []
|
if "signers" not in rules:
|
||||||
rules["signers"].append(user_address) # Add user as valid signer
|
rules["signers"] = []
|
||||||
|
rules["signers"].append(user_address) # Add user as valid signer
|
||||||
rules["user_address"] = user_address
|
rules["user_address"] = user_address
|
||||||
|
|
||||||
return rules
|
return rules
|
||||||
|
@ -197,7 +216,7 @@ class ContentManager(object):
|
||||||
# Return: The new content if filewrite = False
|
# Return: The new content if filewrite = False
|
||||||
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
|
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
|
||||||
content = self.contents.get(inner_path)
|
content = self.contents.get(inner_path)
|
||||||
if not content: # Content not exist yet, load default one
|
if not content: # Content not exist yet, load default one
|
||||||
self.log.info("File %s not exist yet, loading default values..." % inner_path)
|
self.log.info("File %s not exist yet, loading default values..." % inner_path)
|
||||||
content = {"files": {}, "signs": {}} # Default content.json
|
content = {"files": {}, "signs": {}} # Default content.json
|
||||||
if inner_path == "content.json": # It's the root content.json, add some more fields
|
if inner_path == "content.json": # It's the root content.json, add some more fields
|
||||||
|
@ -205,7 +224,8 @@ class ContentManager(object):
|
||||||
content["description"] = ""
|
content["description"] = ""
|
||||||
content["signs_required"] = 1
|
content["signs_required"] = 1
|
||||||
content["ignore"] = ""
|
content["ignore"] = ""
|
||||||
if extend: content.update(extend) # Add custom fields
|
if extend:
|
||||||
|
content.update(extend) # Add custom fields
|
||||||
|
|
||||||
directory = self.toDir(self.site.storage.getPath(inner_path))
|
directory = self.toDir(self.site.storage.getPath(inner_path))
|
||||||
self.log.info("Opening site data directory: %s..." % directory)
|
self.log.info("Opening site data directory: %s..." % directory)
|
||||||
|
@ -217,18 +237,25 @@ class ContentManager(object):
|
||||||
file_path = self.site.storage.getPath("%s/%s" % (root.strip("/"), file_name))
|
file_path = self.site.storage.getPath("%s/%s" % (root.strip("/"), file_name))
|
||||||
file_inner_path = re.sub(re.escape(directory), "", file_path)
|
file_inner_path = re.sub(re.escape(directory), "", file_path)
|
||||||
|
|
||||||
if file_name == "content.json": ignored = True
|
if file_name == "content.json":
|
||||||
elif content.get("ignore") and re.match(content["ignore"], file_inner_path): ignored = True
|
ignored = True
|
||||||
elif file_name.startswith("."): ignored = True
|
elif content.get("ignore") and re.match(content["ignore"], file_inner_path):
|
||||||
else: ignored = False
|
ignored = True
|
||||||
|
elif file_name.startswith("."):
|
||||||
|
ignored = True
|
||||||
|
else:
|
||||||
|
ignored = False
|
||||||
|
|
||||||
if ignored: # Ignore content.json, definied regexp and files starting with .
|
if ignored: # Ignore content.json, definied regexp and files starting with .
|
||||||
self.log.info("- [SKIPPED] %s" % file_inner_path)
|
self.log.info("- [SKIPPED] %s" % file_inner_path)
|
||||||
else:
|
else:
|
||||||
sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
|
sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
|
||||||
self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum))
|
self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum))
|
||||||
hashed_files[file_inner_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
|
hashed_files[file_inner_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
|
||||||
if file_inner_path in content["files"].keys() and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512"):
|
if (
|
||||||
|
file_inner_path in content["files"].keys()
|
||||||
|
and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512")
|
||||||
|
):
|
||||||
changed_files.append(file_path)
|
changed_files.append(file_path)
|
||||||
|
|
||||||
self.log.debug("Changed files: %s" % changed_files)
|
self.log.debug("Changed files: %s" % changed_files)
|
||||||
|
@ -239,9 +266,9 @@ class ContentManager(object):
|
||||||
# Generate new content.json
|
# Generate new content.json
|
||||||
self.log.info("Adding timestamp and sha512sums to new content.json...")
|
self.log.info("Adding timestamp and sha512sums to new content.json...")
|
||||||
|
|
||||||
new_content = content.copy() # Create a copy of current content.json
|
new_content = content.copy() # Create a copy of current content.json
|
||||||
new_content["files"] = hashed_files # Add files sha512 hash
|
new_content["files"] = hashed_files # Add files sha512 hash
|
||||||
new_content["modified"] = time.time() # Add timestamp
|
new_content["modified"] = time.time() # Add timestamp
|
||||||
if inner_path == "content.json":
|
if inner_path == "content.json":
|
||||||
new_content["address"] = self.site.address
|
new_content["address"] = self.site.address
|
||||||
new_content["zeronet_version"] = config.version
|
new_content["zeronet_version"] = config.version
|
||||||
|
@ -255,19 +282,22 @@ class ContentManager(object):
|
||||||
return self.log.error("Private key invalid! Valid signers: %s, Private key address: %s" % (valid_signers, privatekey_address))
|
return self.log.error("Private key invalid! Valid signers: %s, Private key address: %s" % (valid_signers, privatekey_address))
|
||||||
self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers))
|
self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers))
|
||||||
|
|
||||||
if inner_path == "content.json" and privatekey_address == self.site.address: # If signing using the root key sign the valid signers
|
if inner_path == "content.json" and privatekey_address == self.site.address: # If signing using the root key sign the valid signers
|
||||||
new_content["signers_sign"] = CryptBitcoin.sign("%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey)
|
new_content["signers_sign"] = CryptBitcoin.sign("%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey)
|
||||||
if not new_content["signers_sign"]: self.log.info("Old style address, signers_sign is none")
|
if not new_content["signers_sign"]:
|
||||||
|
self.log.info("Old style address, signers_sign is none")
|
||||||
|
|
||||||
self.log.info("Signing %s..." % inner_path)
|
self.log.info("Signing %s..." % inner_path)
|
||||||
|
|
||||||
if "signs" in new_content: del(new_content["signs"]) # Delete old signs
|
if "signs" in new_content:
|
||||||
if "sign" in new_content: del(new_content["sign"]) # Delete old sign (backward compatibility)
|
del(new_content["signs"]) # Delete old signs
|
||||||
|
if "sign" in new_content:
|
||||||
|
del(new_content["sign"]) # Delete old sign (backward compatibility)
|
||||||
|
|
||||||
sign_content = json.dumps(new_content, sort_keys=True)
|
sign_content = json.dumps(new_content, sort_keys=True)
|
||||||
sign = CryptBitcoin.sign(sign_content, privatekey)
|
sign = CryptBitcoin.sign(sign_content, privatekey)
|
||||||
#new_content["signs"] = content.get("signs", {}) # TODO: Multisig
|
# new_content["signs"] = content.get("signs", {}) # TODO: Multisig
|
||||||
if sign: # If signing is successful (not an old address)
|
if sign: # If signing is successful (not an old address)
|
||||||
new_content["signs"] = {}
|
new_content["signs"] = {}
|
||||||
new_content["signs"][privatekey_address] = sign
|
new_content["signs"][privatekey_address] = sign
|
||||||
|
|
||||||
|
@ -294,7 +324,7 @@ class ContentManager(object):
|
||||||
# Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"]
|
# Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"]
|
||||||
def getValidSigners(self, inner_path, content=None):
|
def getValidSigners(self, inner_path, content=None):
|
||||||
valid_signers = []
|
valid_signers = []
|
||||||
if inner_path == "content.json": # Root content.json
|
if inner_path == "content.json": # Root content.json
|
||||||
if "content.json" in self.contents and "signers" in self.contents["content.json"]:
|
if "content.json" in self.contents and "signers" in self.contents["content.json"]:
|
||||||
valid_signers += self.contents["content.json"]["signers"].keys()
|
valid_signers += self.contents["content.json"]["signers"].keys()
|
||||||
else:
|
else:
|
||||||
|
@ -314,11 +344,12 @@ class ContentManager(object):
|
||||||
from Crypt import CryptBitcoin
|
from Crypt import CryptBitcoin
|
||||||
|
|
||||||
rules = self.getRules(inner_path, content)
|
rules = self.getRules(inner_path, content)
|
||||||
if not rules.get("cert_signers"): return True # Does not need cert
|
if not rules.get("cert_signers"):
|
||||||
|
return True # Does not need cert
|
||||||
|
|
||||||
name, domain = content["cert_user_id"].split("@")
|
name, domain = content["cert_user_id"].split("@")
|
||||||
cert_address = rules["cert_signers"].get(domain)
|
cert_address = rules["cert_signers"].get(domain)
|
||||||
if not cert_address: # Cert signer not allowed
|
if not cert_address: # Cert signer not allowed
|
||||||
self.log.error("Invalid cert signer: %s" % domain)
|
self.log.error("Invalid cert signer: %s" % domain)
|
||||||
return False
|
return False
|
||||||
return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"])
|
return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"])
|
||||||
|
@ -326,21 +357,23 @@ class ContentManager(object):
|
||||||
# Checks if the content.json content is valid
|
# Checks if the content.json content is valid
|
||||||
# Return: True or False
|
# Return: True or False
|
||||||
def validContent(self, inner_path, content):
|
def validContent(self, inner_path, content):
|
||||||
content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content
|
content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content
|
||||||
site_size = self.getTotalSize(ignore=inner_path)+content_size # Site size without old content
|
site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content
|
||||||
if site_size > self.site.settings.get("size", 0): self.site.settings["size"] = site_size # Save to settings if larger
|
if site_size > self.site.settings.get("size", 0):
|
||||||
|
self.site.settings["size"] = site_size # Save to settings if larger
|
||||||
|
|
||||||
site_size_limit = self.site.getSizeLimit()*1024*1024
|
site_size_limit = self.site.getSizeLimit() * 1024 * 1024
|
||||||
|
|
||||||
# Check total site size limit
|
# Check total site size limit
|
||||||
if site_size > site_size_limit:
|
if site_size > site_size_limit:
|
||||||
self.log.error("%s: Site too large %s > %s, aborting task..." % (inner_path, site_size, site_size_limit))
|
self.log.error("%s: Site too large %s > %s, aborting task..." % (inner_path, site_size, site_size_limit))
|
||||||
task = self.site.worker_manager.findTask(inner_path)
|
task = self.site.worker_manager.findTask(inner_path)
|
||||||
if task: # Dont try to download from other peers
|
if task: # Dont try to download from other peers
|
||||||
self.site.worker_manager.failTask(task)
|
self.site.worker_manager.failTask(task)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if inner_path == "content.json": return True # Root content.json is passed
|
if inner_path == "content.json":
|
||||||
|
return True # Root content.json is passed
|
||||||
|
|
||||||
# Load include details
|
# Load include details
|
||||||
rules = self.getRules(inner_path, content)
|
rules = self.getRules(inner_path, content)
|
||||||
|
@ -349,15 +382,15 @@ class ContentManager(object):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Check include size limit
|
# Check include size limit
|
||||||
if rules.get("max_size"): # Include size limit
|
if rules.get("max_size"): # Include size limit
|
||||||
if content_size > rules["max_size"]:
|
if content_size > rules["max_size"]:
|
||||||
self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"]))
|
self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"]))
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Check if content includes allowed
|
# Check if content includes allowed
|
||||||
if rules.get("includes_allowed") == False and content.get("includes"):
|
if rules.get("includes_allowed") is False and content.get("includes"):
|
||||||
self.log.error("%s: Includes not allowed" % inner_path)
|
self.log.error("%s: Includes not allowed" % inner_path)
|
||||||
return False # Includes not allowed
|
return False # Includes not allowed
|
||||||
|
|
||||||
# Filename limit
|
# Filename limit
|
||||||
if rules.get("files_allowed"):
|
if rules.get("files_allowed"):
|
||||||
|
@ -370,54 +403,57 @@ class ContentManager(object):
|
||||||
|
|
||||||
# Verify file validity
|
# Verify file validity
|
||||||
# Return: None = Same as before, False = Invalid, True = Valid
|
# Return: None = Same as before, False = Invalid, True = Valid
|
||||||
def verifyFile(self, inner_path, file, ignore_same = True):
|
def verifyFile(self, inner_path, file, ignore_same=True):
|
||||||
if inner_path.endswith("content.json"): # content.json: Check using sign
|
if inner_path.endswith("content.json"): # content.json: Check using sign
|
||||||
from Crypt import CryptBitcoin
|
from Crypt import CryptBitcoin
|
||||||
try:
|
try:
|
||||||
new_content = json.load(file)
|
new_content = json.load(file)
|
||||||
if inner_path in self.contents:
|
if inner_path in self.contents:
|
||||||
old_content = self.contents.get(inner_path)
|
old_content = self.contents.get(inner_path)
|
||||||
# Checks if its newer the ours
|
# Checks if its newer the ours
|
||||||
if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
|
if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
|
||||||
return None
|
return None
|
||||||
elif old_content["modified"] > new_content["modified"]: # We have newer
|
elif old_content["modified"] > new_content["modified"]: # We have newer
|
||||||
self.log.debug("We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"]))
|
self.log.debug("We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"]))
|
||||||
gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers
|
gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers
|
||||||
return False
|
return False
|
||||||
if new_content["modified"] > time.time()+60*60*24: # Content modified in the far future (allow 1 day window)
|
if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day window)
|
||||||
self.log.error("%s modify is in the future!" % inner_path)
|
self.log.error("%s modify is in the future!" % inner_path)
|
||||||
return False
|
return False
|
||||||
# Check sign
|
# Check sign
|
||||||
sign = new_content.get("sign")
|
sign = new_content.get("sign")
|
||||||
signs = new_content.get("signs", {})
|
signs = new_content.get("signs", {})
|
||||||
if "sign" in new_content: del(new_content["sign"]) # The file signed without the sign
|
if "sign" in new_content:
|
||||||
if "signs" in new_content: del(new_content["signs"]) # The file signed without the signs
|
del(new_content["sign"]) # The file signed without the sign
|
||||||
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
|
if "signs" in new_content:
|
||||||
|
del(new_content["signs"]) # The file signed without the signs
|
||||||
|
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
|
||||||
|
|
||||||
if not self.validContent(inner_path, new_content): return False # Content not valid (files too large, invalid files)
|
if not self.validContent(inner_path, new_content):
|
||||||
|
return False # Content not valid (files too large, invalid files)
|
||||||
|
|
||||||
if signs: # New style signing
|
if signs: # New style signing
|
||||||
valid_signers = self.getValidSigners(inner_path, new_content)
|
valid_signers = self.getValidSigners(inner_path, new_content)
|
||||||
signs_required = self.getSignsRequired(inner_path, new_content)
|
signs_required = self.getSignsRequired(inner_path, new_content)
|
||||||
|
|
||||||
if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
|
if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
|
||||||
if not CryptBitcoin.verify("%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]):
|
if not CryptBitcoin.verify("%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]):
|
||||||
self.log.error("%s invalid signers_sign!" % inner_path)
|
self.log.error("%s invalid signers_sign!" % inner_path)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
|
if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
|
||||||
self.log.error("%s invalid cert!" % inner_path)
|
self.log.error("%s invalid cert!" % inner_path)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
valid_signs = 0
|
valid_signs = 0
|
||||||
for address in valid_signers:
|
for address in valid_signers:
|
||||||
if address in signs: valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
|
if address in signs:
|
||||||
if valid_signs >= signs_required: break # Break if we has enough signs
|
valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
|
||||||
|
if valid_signs >= signs_required:
|
||||||
|
break # Break if we has enough signs
|
||||||
|
|
||||||
return valid_signs >= signs_required
|
return valid_signs >= signs_required
|
||||||
else: # Old style signing
|
else: # Old style signing
|
||||||
return CryptBitcoin.verify(sign_content, self.site.address, sign)
|
return CryptBitcoin.verify(sign_content, self.site.address, sign)
|
||||||
|
|
||||||
except Exception, err:
|
except Exception, err:
|
||||||
|
@ -429,7 +465,7 @@ class ContentManager(object):
|
||||||
if file_info:
|
if file_info:
|
||||||
if "sha512" in file_info:
|
if "sha512" in file_info:
|
||||||
hash_valid = CryptHash.sha512sum(file) == file_info["sha512"]
|
hash_valid = CryptHash.sha512sum(file) == file_info["sha512"]
|
||||||
elif "sha1" in file_info: # Backward compatibility
|
elif "sha1" in file_info: # Backward compatibility
|
||||||
hash_valid = CryptHash.sha1sum(file) == file_info["sha1"]
|
hash_valid = CryptHash.sha1sum(file) == file_info["sha1"]
|
||||||
else:
|
else:
|
||||||
hash_valid = False
|
hash_valid = False
|
||||||
|
@ -439,22 +475,21 @@ class ContentManager(object):
|
||||||
return False
|
return False
|
||||||
return hash_valid
|
return hash_valid
|
||||||
|
|
||||||
else: # File not in content.json
|
else: # File not in content.json
|
||||||
self.log.error("File not in content.json: %s" % inner_path)
|
self.log.error("File not in content.json: %s" % inner_path)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
# Get dir from file
|
# Get dir from file
|
||||||
# Return: data/site/content.json -> data/site
|
# Return: data/site/content.json -> data/site
|
||||||
def toDir(self, inner_path):
|
def toDir(self, inner_path):
|
||||||
file_dir = re.sub("[^/]*?$", "", inner_path).strip("/")
|
file_dir = re.sub("[^/]*?$", "", inner_path).strip("/")
|
||||||
if file_dir: file_dir += "/" # Add / at end if its not the root
|
if file_dir:
|
||||||
|
file_dir += "/" # Add / at end if its not the root
|
||||||
return file_dir
|
return file_dir
|
||||||
|
|
||||||
|
|
||||||
def testSign():
|
def testSign():
|
||||||
global config
|
global config
|
||||||
from Config import config
|
|
||||||
from Site import Site
|
from Site import Site
|
||||||
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
||||||
content_manager = ContentManager(site)
|
content_manager = ContentManager(site)
|
||||||
|
@ -462,9 +497,7 @@ def testSign():
|
||||||
|
|
||||||
|
|
||||||
def testVerify():
|
def testVerify():
|
||||||
from Config import config
|
|
||||||
from Site import Site
|
from Site import Site
|
||||||
#site = Site("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr")
|
|
||||||
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
||||||
|
|
||||||
content_manager = ContentManager(site)
|
content_manager = ContentManager(site)
|
||||||
|
@ -478,7 +511,6 @@ def testVerify():
|
||||||
|
|
||||||
|
|
||||||
def testInfo():
|
def testInfo():
|
||||||
from Config import config
|
|
||||||
from Site import Site
|
from Site import Site
|
||||||
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH")
|
||||||
|
|
||||||
|
@ -493,14 +525,13 @@ def testInfo():
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import os, sys, logging
|
import sys
|
||||||
|
import logging
|
||||||
os.chdir("../..")
|
os.chdir("../..")
|
||||||
sys.path.insert(0, os.path.abspath("."))
|
sys.path.insert(0, os.path.abspath("."))
|
||||||
sys.path.insert(0, os.path.abspath("src"))
|
sys.path.insert(0, os.path.abspath("src"))
|
||||||
logging.basicConfig(level=logging.DEBUG)
|
logging.basicConfig(level=logging.DEBUG)
|
||||||
from Debug import Debug
|
|
||||||
from Crypt import CryptHash
|
|
||||||
|
|
||||||
#testSign()
|
# testSign()
|
||||||
testVerify()
|
testVerify()
|
||||||
#testInfo()
|
# testInfo()
|
||||||
|
|
|
@ -185,7 +185,7 @@ class Peer(object):
|
||||||
# Stop and remove from site
|
# Stop and remove from site
|
||||||
def remove(self):
|
def remove(self):
|
||||||
self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed))
|
self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed))
|
||||||
if self.key in self.site.peers: del(self.site.peers[self.key])
|
if self.site and self.key in self.site.peers: del(self.site.peers[self.key])
|
||||||
if self.connection:
|
if self.connection:
|
||||||
self.connection.close()
|
self.connection.close()
|
||||||
|
|
||||||
|
|
|
@ -242,7 +242,12 @@ class Wrapper
|
||||||
@setSiteInfo site_info
|
@setSiteInfo site_info
|
||||||
|
|
||||||
if site_info.settings.size > site_info.size_limit*1024*1024 # Site size too large and not displaying it yet
|
if site_info.settings.size > site_info.size_limit*1024*1024 # Site size too large and not displaying it yet
|
||||||
@loading.showTooLarge(site_info)
|
if @loading.screen_visible
|
||||||
|
@loading.showTooLarge(site_info)
|
||||||
|
else
|
||||||
|
@displayConfirm "Site is larger than allowed: #{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB", "Set limit to #{site_info.next_size_limit}MB", =>
|
||||||
|
@ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) =>
|
||||||
|
@notifications.add("size_limit", "done", res, 5000)
|
||||||
|
|
||||||
if site_info.content
|
if site_info.content
|
||||||
window.document.title = site_info.content.title+" - ZeroNet"
|
window.document.title = site_info.content.title+" - ZeroNet"
|
||||||
|
@ -286,8 +291,8 @@ class Wrapper
|
||||||
@loading.printLine "No peers found"
|
@loading.printLine "No peers found"
|
||||||
|
|
||||||
if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage
|
if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage
|
||||||
if site_info.size_limit < site_info.next_size_limit # Need upgrade soon
|
if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon
|
||||||
@wrapperConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", =>
|
@actionConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", =>
|
||||||
@ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) =>
|
@ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) =>
|
||||||
@notifications.add("size_limit", "done", res, 5000)
|
@notifications.add("size_limit", "done", res, 5000)
|
||||||
return false
|
return false
|
||||||
|
|
|
@ -681,7 +681,6 @@ jQuery.extend( jQuery.easing,
|
||||||
}).call(this);
|
}).call(this);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* ---- src/Ui/media/Sidebar.coffee ---- */
|
/* ---- src/Ui/media/Sidebar.coffee ---- */
|
||||||
|
|
||||||
|
|
||||||
|
@ -1069,7 +1068,15 @@ jQuery.extend( jQuery.easing,
|
||||||
_this.address = site_info.address;
|
_this.address = site_info.address;
|
||||||
_this.setSiteInfo(site_info);
|
_this.setSiteInfo(site_info);
|
||||||
if (site_info.settings.size > site_info.size_limit * 1024 * 1024) {
|
if (site_info.settings.size > site_info.size_limit * 1024 * 1024) {
|
||||||
_this.loading.showTooLarge(site_info);
|
if (_this.loading.screen_visible) {
|
||||||
|
_this.loading.showTooLarge(site_info);
|
||||||
|
} else {
|
||||||
|
_this.displayConfirm("Site is larger than allowed: " + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB", "Set limit to " + site_info.next_size_limit + "MB", function() {
|
||||||
|
return _this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) {
|
||||||
|
return _this.notifications.add("size_limit", "done", res, 5000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (site_info.content) {
|
if (site_info.content) {
|
||||||
window.document.title = site_info.content.title + " - ZeroNet";
|
window.document.title = site_info.content.title + " - ZeroNet";
|
||||||
|
@ -1118,8 +1125,8 @@ jQuery.extend( jQuery.easing,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) {
|
if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) {
|
||||||
if (site_info.size_limit < site_info.next_size_limit) {
|
if (site_info.size_limit * 1.1 < site_info.next_size_limit) {
|
||||||
this.wrapperConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) {
|
this.actionConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) {
|
||||||
return function() {
|
return function() {
|
||||||
_this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) {
|
_this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) {
|
||||||
return _this.notifications.add("size_limit", "done", res, 5000);
|
return _this.notifications.add("size_limit", "done", res, 5000);
|
||||||
|
|
Loading…
Reference in a new issue