diff --git a/src/Content/ContentDb.py b/src/Content/ContentDb.py index 1f74222a..83ab59ad 100644 --- a/src/Content/ContentDb.py +++ b/src/Content/ContentDb.py @@ -62,14 +62,20 @@ class ContentDb(Db): def initSite(self, site): self.sites[site.address] = site + def needSite(self, site): + if site.address not in self.site_ids: + self.execute("INSERT OR IGNORE INTO site ?", {"address": site.address}) + self.site_ids = {} for row in self.execute("SELECT * FROM site"): self.site_ids[row["address"]] = row["site_id"] - return self.site_ids[site_address] + return self.site_ids[site.address] - def deleteSite(self, site_address): - site_id = self.site_ids[site_address] - self.execute("DELETE FROM site WHERE site_id = :site_id", {"site_id": site_id}) - del self.site_ids[site_address] + def deleteSite(self, site): + site_id = self.site_ids.get(site.address, 0) + if site_id: + self.execute("DELETE FROM site WHERE site_id = :site_id", {"site_id": site_id}) + del self.site_ids[site.address] + del self.sites[site.address] def setContent(self, site, inner_path, content, size=0): self.insertOrUpdate("content", { @@ -82,13 +88,13 @@ class ContentDb(Db): "inner_path": inner_path }) - def deleteContent(self, site_address, inner_path): - self.execute("DELETE FROM content WHERE ?", {"site_id": self.site_ids[site_address], "inner_path": inner_path}) + def deleteContent(self, site, inner_path): + self.execute("DELETE FROM content WHERE ?", {"site_id": self.site_ids.get(site.address, 0), "inner_path": inner_path}) - def loadDbDict(self, site_address): + def loadDbDict(self, site): res = self.execute( "SELECT GROUP_CONCAT(inner_path, '|') AS inner_paths FROM content WHERE ?", - {"site_id": self.site_ids[site_address]} + {"site_id": self.site_ids.get(site.address, 0)} ) row = res.fetchone() if row and row["inner_paths"]: @@ -97,24 +103,24 @@ class ContentDb(Db): else: return {} - def getTotalSize(self, site_address, ignore=None): - params = {"site_id": self.site_ids[site_address]} + def getTotalSize(self, site, ignore=None): + params = {"site_id": self.site_ids.get(site.address, 0)} if ignore: params["not__inner_path"] = ignore res = self.execute("SELECT SUM(size) + SUM(size_files) AS size FROM content WHERE ?", params) return res.fetchone()["size"] - def getOptionalSize(self, site_address): + def getOptionalSize(self, site): res = self.execute( "SELECT SUM(size_files_optional) AS size FROM content WHERE ?", - {"site_id": self.site_ids[site_address]} + {"site_id": self.site_ids.get(site.address, 0)} ) return res.fetchone()["size"] - def listModified(self, site_address, since): + def listModified(self, site, since): res = self.execute( "SELECT inner_path, modified FROM content WHERE site_id = :site_id AND modified > :since", - {"site_id": self.site_ids[site_address], "since": since} + {"site_id": self.site_ids.get(site.address, 0), "since": since} ) return {row["inner_path"]: row["modified"] for row in res} diff --git a/src/Content/ContentDbDict.py b/src/Content/ContentDbDict.py index 19ba13cf..4e1551ee 100644 --- a/src/Content/ContentDbDict.py +++ b/src/Content/ContentDbDict.py @@ -3,18 +3,16 @@ import os import ContentDb - class ContentDbDict(dict): def __init__(self, site, *args, **kwargs): s = time.time() self.site = site - self.site_address = site.address self.cached_keys = [] self.log = self.site.log self.db = ContentDb.getContentDb() - self.db_id = self.db.needSite(site.address) + self.db_id = self.db.needSite(site) self.num_loaded = 0 - super(ContentDbDict, self).__init__(self.db.loadDbDict(site.address)) # Load keys from database + super(ContentDbDict, self).__init__(self.db.loadDbDict(site)) # Load keys from database self.log.debug("ContentDb init: %.3fs, found files: %s, sites: %s" % (time.time() - s, len(self), len(self.db.site_ids))) def loadItem(self, key): @@ -25,7 +23,8 @@ class ContentDbDict(dict): content = self.site.storage.loadJson(key) dict.__setitem__(self, key, content) except IOError: - self.__delitem__(key) # File not exists anymore + if dict.get(self, key): + self.__delitem__(key) # File not exists anymore raise KeyError(key) self.addCachedKey(key) @@ -59,15 +58,17 @@ class ContentDbDict(dict): dict.__setitem__(self, key, val) self.addCachedKey(key) self.checkLimit() - self.db.setContent(self.site_address, key, val, size=self.getItemSize(key)) + size = self.getItemSize(key) + self.db.setContent(self.site, key, val, size) + dict.__setitem__(self, key, val) def __delitem__(self, key): + self.db.deleteContent(self.site, key) dict.__delitem__(self, key) try: self.cached_keys.remove(key) except ValueError: pass - self.db.deleteContent(self.site_address, key) def iteritems(self): for key in dict.keys(self):