rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
This commit is contained in:
parent
34b7cb0292
commit
eddf3eb8fc
14 changed files with 286 additions and 248 deletions
36
src/util/StreamingMsgpack.py
Normal file
36
src/util/StreamingMsgpack.py
Normal file
|
@ -0,0 +1,36 @@
|
|||
import msgpack, os, struct
|
||||
|
||||
|
||||
def msgpackHeader(size):
|
||||
if size <= 2**8-1:
|
||||
return b"\xc4" + struct.pack("B", size)
|
||||
elif size <= 2**16-1:
|
||||
return b"\xc5" + struct.pack(">H", size)
|
||||
elif size <= 2**32-1:
|
||||
return b"\xc6" + struct.pack(">I", size)
|
||||
else:
|
||||
raise Exception("huge binary string")
|
||||
|
||||
|
||||
def stream(data, writer):
|
||||
packer = msgpack.Packer()
|
||||
writer(packer.pack_map_header(len(data)))
|
||||
for key, val in data.iteritems():
|
||||
writer(packer.pack(key))
|
||||
if issubclass(type(val), file): # File obj
|
||||
max_size = os.fstat(val.fileno()).st_size-val.tell()
|
||||
size = min(max_size, val.read_bytes)
|
||||
bytes_left = size
|
||||
writer(msgpackHeader(size))
|
||||
buff = 1024*64
|
||||
while 1:
|
||||
writer(val.read(min(bytes_left, buff)))
|
||||
bytes_left = bytes_left-buff
|
||||
if bytes_left <= 0: break
|
||||
else: # Simple
|
||||
writer(packer.pack(val))
|
||||
return size
|
||||
|
||||
|
||||
class FilePart(file):
|
||||
pass
|
Loading…
Add table
Add a link
Reference in a new issue