oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use separate db connection for rebuilding

Browse source
This commit is contained in:
shortcutme 2018-01-19 02:31:46 +01:00
parent ea1cd63929
commit 005358f4bf
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
1 changed files with 33 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
src/Site/SiteStorage.py
View file

@ -43,25 +43,11 @@ class SiteStorage(object):
else: else:
return False return False
# Load db from dbschema.json # Create new databaseobject with the site's schema
def openDb(self, check=True): def openDb(self):
try: schema = self.getDbSchema()
schema = self.loadJson("dbschema.json") db_path = self.getPath(schema["db_file"])
db_path = self.getPath(schema["db_file"]) return Db(schema, db_path)
except Exception, err:
raise Exception("dbschema.json is not a valid JSON: %s" % err)
if check:
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: # Not exist or null
self.rebuildDb()
if not self.db:
self.db = Db(schema, db_path)
if check and not self.db_checked:
changed_tables = self.db.checkTables()
if changed_tables:
self.rebuildDb(delete_db=False) # TODO: only update the changed table datas
def closeDb(self): def closeDb(self):
if self.db: if self.db:
@ -69,6 +55,13 @@ class SiteStorage(object):
self.event_db_busy = None self.event_db_busy = None
self.db = None self.db = None
def getDbSchema(self):
try:
schema = self.loadJson("dbschema.json")
except Exception, err:
raise Exception("dbschema.json is not a valid JSON: %s" % err)
return schema
# Return db class # Return db class
def getDb(self): def getDb(self):
if not self.db: if not self.db:
@ -76,7 +69,19 @@ class SiteStorage(object):
self.site.needFile("dbschema.json", priority=3) self.site.needFile("dbschema.json", priority=3)
self.has_db = self.isFile("dbschema.json") # Recheck if dbschema exist self.has_db = self.isFile("dbschema.json") # Recheck if dbschema exist
if self.has_db: if self.has_db:
self.openDb() schema = self.getDbSchema()
db_path = self.getPath(schema["db_file"])
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
self.rebuildDb()
if self.db:
self.db.close()
self.db = self.openDb()
changed_tables = self.db.checkTables()
if changed_tables:
self.rebuildDb(delete_db=False) # TODO: only update the changed table datas
return self.db return self.db
def updateDbFile(self, inner_path, file=None, cur=None): def updateDbFile(self, inner_path, file=None, cur=None):
@ -124,11 +129,11 @@ class SiteStorage(object):
os.unlink(db_path) os.unlink(db_path)
except Exception, err: except Exception, err:
self.log.error("Delete error: %s" % err) self.log.error("Delete error: %s" % err)
self.db = None
self.openDb(check=False) db = self.openDb()
self.log.info("Creating tables...") self.log.info("Creating tables...")
self.db.checkTables() db.checkTables()
cur = self.db.getCursor() cur = db.getCursor()
cur.execute("BEGIN") cur.execute("BEGIN")
cur.logging = False cur.logging = False
found = 0 found = 0
@ -155,6 +160,9 @@ class SiteStorage(object):
finally: finally:
cur.execute("END") cur.execute("END")
cur.close()
db.close()
self.log.info("Closing Db: %s" % db)
if len(db_files) > 100: if len(db_files) > 100:
self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files..."].format(found, len(db_files)), "rebuild", 100) self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files..."].format(found, len(db_files)), "rebuild", 100)
self.log.info("Imported %s data file in %ss" % (found, time.time() - s)) self.log.info("Imported %s data file in %ss" % (found, time.time() - s))
@ -278,7 +286,7 @@ class SiteStorage(object):
# Reopen DB to check changes # Reopen DB to check changes
if self.has_db: if self.has_db:
self.closeDb() self.closeDb()
self.openDb() self.getDb()
elif not config.disable_db and (inner_path.endswith(".json") or inner_path.endswith(".json.gz")) and self.has_db: # Load json file to db elif not config.disable_db and (inner_path.endswith(".json") or inner_path.endswith(".json.gz")) and self.has_db: # Load json file to db
if config.verbose: if config.verbose:
self.log.debug("Loading json file to db: %s (file: %s)" % (inner_path, file)) self.log.debug("Loading json file to db: %s (file: %s)" % (inner_path, file))