oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Db rebuilding error display, reconnect bug fix

Browse source
This commit is contained in:
shortcutme 2019-03-16 00:11:52 +01:00
parent 59426c31f7
commit 002303a765
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
1 changed files with 26 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
src/Site/SiteStorage.py
View file

@ -116,15 +116,16 @@ class SiteStorage(object):
# Rebuild sql cache # Rebuild sql cache
@util.Noparallel() @util.Noparallel()
def rebuildDb(self, delete_db=True): def rebuildDb(self, delete_db=True):
self.log.info("Rebuilding db...")
self.has_db = self.isFile("dbschema.json") self.has_db = self.isFile("dbschema.json")
if not self.has_db: if not self.has_db:
return False return False
self.event_db_busy = gevent.event.AsyncResult()
schema = self.loadJson("dbschema.json") schema = self.loadJson("dbschema.json")
db_path = self.getPath(schema["db_file"]) db_path = self.getPath(schema["db_file"])
if os.path.isfile(db_path) and delete_db: if os.path.isfile(db_path) and delete_db:
if self.db: if self.db:
self.db.close() # Close db if open self.closeDb() # Close db if open
time.sleep(0.5) time.sleep(0.5)
self.log.info("Deleting %s" % db_path) self.log.info("Deleting %s" % db_path)
try: try:
@ -132,42 +133,46 @@ class SiteStorage(object):
except Exception as err: except Exception as err:
self.log.error("Delete error: %s" % err) self.log.error("Delete error: %s" % err)
db = self.openDb() if not self.db:
self.db = self.openDb()
self.event_db_busy = gevent.event.AsyncResult()
self.log.info("Creating tables...") self.log.info("Creating tables...")
db.checkTables() self.db.checkTables()
cur = db.getCursor() cur = self.db.getCursor()
cur.execute("BEGIN")
cur.logging = False cur.logging = False
found = 0
s = time.time() s = time.time()
self.log.info("Getting db files...") self.log.info("Getting db files...")
db_files = list(self.getDbFiles()) db_files = list(self.getDbFiles())
num_imported = 0
num_total = len(db_files)
num_error = 0
self.log.info("Importing data...") self.log.info("Importing data...")
try: try:
if len(db_files) > 100: if num_total > 100:
self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files..."].format("0000", len(db_files)), "rebuild", 0) self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files (error: {2})..."].format("0000", num_total, num_error), "rebuild", 0)
for file_inner_path, file_path in db_files: for file_inner_path, file_path in db_files:
try: try:
if self.updateDbFile(file_inner_path, file=open(file_path, "rb"), cur=cur): if self.updateDbFile(file_inner_path, file=open(file_path, "rb"), cur=cur):
found += 1 num_imported += 1
except Exception, err: except Exception as err:
self.log.error("Error importing %s: %s" % (file_inner_path, Debug.formatException(err))) self.log.error("Error importing %s: %s" % (file_inner_path, Debug.formatException(err)))
if found and found % 100 == 0: num_error += 1
if num_imported and num_imported % 100 == 0:
self.site.messageWebsocket( self.site.messageWebsocket(
_["Database rebuilding...<br>Imported {0} of {1} files..."].format(found, len(db_files)), _["Database rebuilding...<br>Imported {0} of {1} files (error: {2})..."].format(num_imported, num_total, num_error),
"rebuild", "rebuild",
int(float(found) / len(db_files) * 100) int(float(num_imported) / num_total * 100)
) )
time.sleep(0.000001) # Context switch to avoid UI block time.sleep(0.001) # Context switch to avoid UI block
finally: finally:
cur.execute("END")
cur.close() cur.close()
db.close() if num_total > 100:
self.log.info("Closing Db: %s" % db) self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files (error: {2})..."].format(num_imported, num_total, num_error), "rebuild", 100)
if len(db_files) > 100: self.log.info("Imported %s data file in %.3fs" % (num_imported, time.time() - s))
self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files..."].format(found, len(db_files)), "rebuild", 100)
self.log.info("Imported %s data file in %ss" % (found, time.time() - s))
self.event_db_busy.set(True) # Event done, notify waiters self.event_db_busy.set(True) # Event done, notify waiters
self.event_db_busy = None # Clear event self.event_db_busy = None # Clear event