Merge pull request #2642 from imachug/better-logs

Use more unique yet short paths for logging

src/Debug/Debug.py

@@ -28,7 +28,12 @@ def formatExceptionMessage(err):
     return "%s: %s" % (err_type, err_message)
 
 
-python_lib_dir = os.path.dirname(os.__file__)
+python_lib_dirs = [path for path in sys.path if path.endswith("/site-packages") or path.endswith("/dist-packages")]
+python_lib_dirs.append(os.path.dirname(os.__file__))  # TODO: check if returns the correct path for PyPy
+
+root_dir = os.path.realpath(__file__)
+for _ in range(3):
+    root_dir = os.path.dirname(root_dir)
 
 
 def formatTraceback(items, limit=None, fold_builtin=True):
@@ -40,30 +45,63 @@ def formatTraceback(items, limit=None, fold_builtin=True):
     for path, line in items:
         i += 1
         is_last = i == len(items)
-        dir_name, file_name = os.path.split(path.replace("\\", "/"))
+        path = path.replace("\\", "/")
 
-        plugin_match = re.match(".*/plugins/(.+)$", dir_name)
+        if path.startswith("src/gevent/"):
-        if plugin_match:
+            file_title = "<gevent>/" + path[len("src/gevent/"):]
-            file_title = "%s/%s" % (plugin_match.group(1), file_name)
+            is_builtin = True
-            is_prev_builtin = False
+            is_skippable_builtin = False
-        elif path.startswith(python_lib_dir):
+        elif path in ("<frozen importlib._bootstrap>", "<frozen importlib._bootstrap_external>"):
-            if is_prev_builtin and not is_last and fold_builtin:
+            file_title = "(importlib)"
-                if back[-1] != "...":
+            is_builtin = True
-                    back.append("...")
+            is_skippable_builtin = True
-                continue
+        else:
+            is_skippable_builtin = False
+            for base in python_lib_dirs:
+                if path.startswith(base + "/"):
+                    file_title = path[len(base + "/"):]
+                    module_name, *tail = file_title.split("/")
+                    if module_name.endswith(".py"):
+                        module_name = module_name[:-3]
+                    file_title = "/".join(["<%s>" % module_name] + tail)
+                    is_builtin = True
+                    break
             else:
-                file_title = path.replace(python_lib_dir, "").replace("\\", "/").strip("/").replace("site-packages/", "")
+                is_builtin = False
-            is_prev_builtin = True
+                for base in (root_dir + "/src", root_dir + "/plugins", root_dir):
-        else:
+                    if path.startswith(base + "/"):
-            file_title = file_name
+                        file_title = path[len(base + "/"):]
-            is_prev_builtin = False
+                        break
+                else:
+                    # For unknown paths, do our best to hide absolute path
+                    file_title = path
+                    for needle in ("/zeronet/", "/core/"):
+                        if needle in file_title.lower():
+                            file_title = "?/" + file_title[file_title.lower().rindex(needle) + len(needle):]
 
-        if file_title == prev_file_title:
+        # Path compression: A/AB/ABC/X/Y.py -> ABC/X/Y.py
-            back.append("%s" % line)
+        # E.g.: in 'Db/DbCursor.py' the directory part is unnecessary
+        if not file_title.startswith("/"):
+            prev_part = ""
+            for i, part in enumerate(file_title.split("/") + [""]):
+                if not part.startswith(prev_part):
+                    break
+                prev_part = part
+            file_title = "/".join(file_title.split("/")[i - 1:])
+
+        if is_skippable_builtin and fold_builtin:
+            pass
+        elif is_builtin and is_prev_builtin and not is_last and fold_builtin:
+            if back[-1] != "...":
+                back.append("...")
         else:
-            back.append("%s line %s" % (file_title, line))
+            if file_title == prev_file_title:
+                back.append("%s" % line)
+            else:
+                back.append("%s line %s" % (file_title, line))
 
         prev_file_title = file_title
+        is_prev_builtin = is_builtin
 
         if limit and i >= limit:
             back.append("...")

src/Test/TestDebug.py

@@ -0,0 +1,53 @@
+from Debug import Debug
+import gevent
+import os
+
+import pytest
+
+
+class TestDebug:
+    @pytest.mark.parametrize("items,expected", [
+        (["@/src/A/B/C.py:17"], ["A/B/C.py line 17"]),  # basic test
+        (["@/src/Db/Db.py:17"], ["Db.py line 17"]),  # path compression
+        (["%s:1" % __file__], ["TestDebug.py line 1"]),
+        (["@/plugins/Chart/ChartDb.py:100"], ["ChartDb.py line 100"]),  # plugins
+        (["@/main.py:17"], ["main.py line 17"]),  # root
+        (["@\\src\\Db\\__init__.py:17"], ["Db/__init__.py line 17"]),  # Windows paths
+        (["<frozen importlib._bootstrap>:1"], []),  # importlib builtins
+        (["<frozen importlib._bootstrap_external>:1"], []),  # importlib builtins
+        (["/home/ivanq/ZeroNet/src/main.py:13"], ["?/src/main.py line 13"]),  # best-effort anonymization
+        (["C:\\ZeroNet\\core\\src\\main.py:13"], ["?/src/main.py line 13"]),
+        (["/root/main.py:17"], ["/root/main.py line 17"]),
+        (["{gevent}:13"], ["<gevent>/__init__.py line 13"]),  # modules
+        (["{os}:13"], ["<os> line 13"]),  # python builtin modules
+        (["src/gevent/event.py:17"], ["<gevent>/event.py line 17"]),  # gevent-overriden __file__
+        (["@/src/Db/Db.py:17", "@/src/Db/DbQuery.py:1"], ["Db.py line 17", "DbQuery.py line 1"]),  # mutliple args
+        (["@/src/Db/Db.py:17", "@/src/Db/Db.py:1"], ["Db.py line 17", "1"]),  # same file
+        (["{os}:1", "@/src/Db/Db.py:17"], ["<os> line 1", "Db.py line 17"]),  # builtins
+        (["{gevent}:1"] + ["{os}:3"] * 4 + ["@/src/Db/Db.py:17"], ["<gevent>/__init__.py line 1", "...", "Db.py line 17"])
+    ])
+    def testFormatTraceback(self, items, expected):
+        q_items = []
+        for item in items:
+            file, line = item.rsplit(":", 1)
+            if file.startswith("@"):
+                file = Debug.root_dir + file[1:]
+            file = file.replace("{os}", os.__file__)
+            file = file.replace("{gevent}", gevent.__file__)
+            q_items.append((file, int(line)))
+        assert Debug.formatTraceback(q_items) == expected
+
+
+    def testFormatException(self):
+        try:
+            raise ValueError("Test exception")
+        except:
+            assert Debug.formatException() == "ValueError: Test exception in TestDebug.py line 43"
+        try:
+            os.path.abspath(1)
+        except:
+            assert "in TestDebug.py line 47 > <posixpath> line " in Debug.formatException()
+
+
+    def testFormatStack(self):
+        assert Debug.formatStack().startswith("TestDebug.py line 53 > <_pytest>/python.py line ")