.forgejo/workflows/build-on-commit.yml

@@ -1,40 +0,0 @@
-name: Build Docker Image on Commit
-
-on:
-  push:
-    branches:
-      - main
-    tags: 
-      - '!' # Exclude tags
-
-jobs:
-  build-and-publish:
-    runs-on: docker-builder
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set REPO_VARS
-        id: repo-url
-        run: |
-          echo "REPO_HOST=$(echo "${{ github.server_url }}" | sed 's~http[s]*://~~g')" >> $GITHUB_ENV
-          echo "REPO_PATH=${{ github.repository }}" >> $GITHUB_ENV
-
-      - name: Login to OCI registry
-        run: |
-          echo "${{ secrets.OCI_TOKEN }}" | docker login $REPO_HOST -u "${{ secrets.OCI_USER }}" --password-stdin
-
-      - name: Build and push Docker images
-        run: |
-          # Build Docker image with commit SHA
-          docker build -t $REPO_HOST/$REPO_PATH:${{ github.sha }} .
-          docker push $REPO_HOST/$REPO_PATH:${{ github.sha }}
-
-          # Build Docker image with nightly tag
-          docker tag $REPO_HOST/$REPO_PATH:${{ github.sha }} $REPO_HOST/$REPO_PATH:nightly
-          docker push $REPO_HOST/$REPO_PATH:nightly
-
-          # Remove local images to save storage
-          docker rmi $REPO_HOST/$REPO_PATH:${{ github.sha }}
-          docker rmi $REPO_HOST/$REPO_PATH:nightly

.forgejo/workflows/build-on-tag.yml

@@ -1,37 +0,0 @@
-name: Build and Publish Docker Image on Tag
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build-and-publish:
-    runs-on: docker-builder
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set REPO_VARS
-        id: repo-url
-        run: |
-          echo "REPO_HOST=$(echo "${{ github.server_url }}" | sed 's~http[s]*://~~g')" >> $GITHUB_ENV
-          echo "REPO_PATH=${{ github.repository }}" >> $GITHUB_ENV
-
-      - name: Login to OCI registry
-        run: |
-          echo "${{ secrets.OCI_TOKEN }}" | docker login $REPO_HOST -u "${{ secrets.OCI_USER }}" --password-stdin
-
-      - name: Build and push Docker image
-        run: |
-          TAG=${{ github.ref_name }} # Get the tag name from the context
-          # Build and push multi-platform Docker images
-          docker build -t $REPO_HOST/$REPO_PATH:$TAG --push .
-          # Tag and push latest
-          docker tag $REPO_HOST/$REPO_PATH:$TAG $REPO_HOST/$REPO_PATH:latest
-          docker push $REPO_HOST/$REPO_PATH:latest
-
-          # Remove the local image to save storage
-          docker rmi $REPO_HOST/$REPO_PATH:$TAG
-          docker rmi $REPO_HOST/$REPO_PATH:latest

.github/FUNDING.yml

@@ -1,10 +0,0 @@
-github: canewsin
-patreon: # Replace with a single Patreon username e.g., user1
-open_collective: # Replace with a single Open Collective username e.g., user1
-ko_fi: canewsin
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: canewsin
-issuehunt: # Replace with a single IssueHunt username e.g., user1
-otechie: # Replace with a single Otechie username e.g., user1
-custom: ['https://paypal.me/PramUkesh', 'https://zerolink.ml/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/']

.github/workflows/codeql-analysis.yml

@@ -1,72 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ py3-latest ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ py3-latest ]
-  schedule:
-    - cron: '32 19 * * 2'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript', 'python' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-        
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

.github/workflows/tests.yml

@@ -1,51 +0,0 @@
-name: tests
-
-on: [push, pull_request]
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    strategy:
-      max-parallel: 16
-      matrix:
-        python-version: ["3.7", "3.8", "3.9"]
-
-    steps:
-      - name: Checkout ZeroNet
-        uses: actions/checkout@v2
-        with:
-          submodules: "true"
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Prepare for installation
-        run: |
-          python3 -m pip install setuptools
-          python3 -m pip install --upgrade pip wheel
-          python3 -m pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
-
-      - name: Install
-        run: |
-          python3 -m pip install --upgrade -r requirements.txt
-          python3 -m pip list
-
-      - name: Prepare for tests
-        run: |
-          openssl version -a
-          echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6
-
-      - name: Test
-        run: |
-          catchsegv python3 -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini
-          export ZERONET_LOG_DIR="log/CryptMessage"; catchsegv python3 -m pytest -x plugins/CryptMessage/Test
-          export ZERONET_LOG_DIR="log/Bigfile"; catchsegv python3 -m pytest -x plugins/Bigfile/Test
-          export ZERONET_LOG_DIR="log/AnnounceLocal"; catchsegv python3 -m pytest -x plugins/AnnounceLocal/Test
-          export ZERONET_LOG_DIR="log/OptionalManager"; catchsegv python3 -m pytest -x plugins/OptionalManager/Test
-          export ZERONET_LOG_DIR="log/Multiuser"; mv plugins/disabled-Multiuser plugins/Multiuser && catchsegv python -m pytest -x plugins/Multiuser/Test
-          export ZERONET_LOG_DIR="log/Bootstrapper"; mv plugins/disabled-Bootstrapper plugins/Bootstrapper && catchsegv python -m pytest -x plugins/Bootstrapper/Test
-          find src -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
-          find plugins -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
-          flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/

.gitignore

@@ -7,11 +7,8 @@ __pycache__/
 
 # Hidden files
 .*
-!/.forgejo
-!/.github
 !/.gitignore
 !/.travis.yml
-!/.gitlab-ci.yml
 
 # Temporary files
 *.bak
@@ -31,6 +28,3 @@ tools/phantomjs
 
 # ZeroNet config file
 zeronet.conf
-
-# ZeroNet log files
-log/*

.gitlab-ci.yml

@@ -1,48 +0,0 @@
-stages:
-  - test
-
-.test_template: &test_template
-  stage: test
-  before_script:
-    - pip install --upgrade pip wheel
-    # Selenium and requests can't be installed without a requests hint on Python 3.4
-    - pip install --upgrade requests>=2.22.0
-    - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
-    - pip install --upgrade -r requirements.txt
-  script:
-    - pip list
-    - openssl version -a
-    - python -m pytest -x plugins/CryptMessage/Test --color=yes
-    - python -m pytest -x plugins/Bigfile/Test --color=yes
-    - python -m pytest -x plugins/AnnounceLocal/Test --color=yes
-    - python -m pytest -x plugins/OptionalManager/Test --color=yes
-    - python -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini --color=yes
-    - mv plugins/disabled-Multiuser plugins/Multiuser
-    - python -m pytest -x plugins/Multiuser/Test --color=yes
-    - mv plugins/disabled-Bootstrapper plugins/Bootstrapper
-    - python -m pytest -x plugins/Bootstrapper/Test --color=yes
-    - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/
-
-test:py3.4:
-  image: python:3.4.3
-  <<: *test_template
-
-test:py3.5:
-  image: python:3.5.7
-  <<: *test_template
-
-test:py3.6:
-  image: python:3.6.9
-  <<: *test_template
-
-test:py3.7-openssl1.1.0:
-  image: python:3.7.0b5
-  <<: *test_template
-
-test:py3.7-openssl1.1.1:
-  image: python:3.7.4
-  <<: *test_template
-
-test:py3.8:
-  image: python:3.8.0b3
-  <<: *test_template

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "plugins"]
-	path = plugins
-	url = https://github.com/ZeroNetX/ZeroNet-Plugins.git

.travis.yml

@@ -4,17 +4,19 @@ python:
   - 3.5
   - 3.6
   - 3.7
-  - 3.8
+  - 3.8-dev
+dist: xenial
 services:
  - docker
 cache: pip
 before_install:
   - pip install --upgrade pip wheel
-  - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
+  - pip install codecov coveralls flake8 mock pytest pytest-cov selenium
   # - docker build -t zeronet .
   # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet
 install:
- - pip install --upgrade -r requirements.txt
+ - if [[ $TRAVIS_PYTHON_VERSION == 3.8-dev ]]; then pip install setuptools cffi 'cython>=0.28' git+git://github.com/gevent/gevent.git#egg=gevent; fi
+ - pip install -r requirements.txt
  - pip list
 before_script:
   - openssl version -a
@@ -24,19 +26,14 @@ before_script:
       sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6';
     fi
 script:
- - catchsegv python -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini
- - export ZERONET_LOG_DIR="log/CryptMessage"; catchsegv python -m pytest -x plugins/CryptMessage/Test
- - export ZERONET_LOG_DIR="log/Bigfile"; catchsegv python -m pytest -x plugins/Bigfile/Test
- - export ZERONET_LOG_DIR="log/AnnounceLocal"; catchsegv python -m pytest -x plugins/AnnounceLocal/Test
- - export ZERONET_LOG_DIR="log/OptionalManager"; catchsegv python -m pytest -x plugins/OptionalManager/Test
- - export ZERONET_LOG_DIR="log/Multiuser"; mv plugins/disabled-Multiuser plugins/Multiuser && catchsegv python -m pytest -x plugins/Multiuser/Test
- - export ZERONET_LOG_DIR="log/Bootstrapper"; mv plugins/disabled-Bootstrapper plugins/Bootstrapper && catchsegv python -m pytest -x plugins/Bootstrapper/Test
- - find src -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
- - find plugins -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
- - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/
-after_failure:
-  - zip -r log.zip log/
-  - curl --upload-file ./log.zip https://transfer.sh/log.zip 
+ - python -m pytest -x plugins/CryptMessage/Test
+ - python -m pytest -x plugins/Bigfile/Test
+ - python -m pytest -x plugins/AnnounceLocal/Test
+ - python -m pytest -x plugins/OptionalManager/Test
+ - python -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini
+ - mv plugins/disabled-Multiuser plugins/Multiuser && python -m pytest -x plugins/Multiuser/Test
+ - mv plugins/disabled-Bootstrapper plugins/Bootstrapper && python -m pytest -x plugins/Bootstrapper/Test
+ - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pybitcointools/  
 after_success:
   - codecov
   - coveralls --rcfile=src/Test/coverage.ini

CHANGELOG-zh-cn.md

@@ -0,0 +1,134 @@
+## ZeroNet 0.5.1 (2016-11-18)
+### 新增
+- 多语言界面
+- 新插件：为站点 HTML 与 JS 文件提供的翻译助手
+- 每个站点独立的 favicon
+
+### 修复
+- 并行可选文件下载
+
+## ZeroNet 0.5.0 (2016-11-08)
+### 新增
+- 新插件：允许在 ZeroHello 列出/删除/固定/管理文件
+- 新的 API 命令来关注用户的可选文件，与可选文件的请求统计
+- 新的可选文件总大小限制
+- 新插件：保存节点到数据库并在重启时保持它们，使得更快的可选文件搜索以及在没有 Tracker 的情况下工作
+- 重写 UPnP 端口打开器 + 退出时关闭端口（感谢 sirMackk!）
+- 通过懒惰 PeerHashfield 创建来减少内存占用
+- 在 /Stats 页面加载 JSON 文件统计与数据库信息
+
+### 更改
+- 独立的锁定文件来获得更好的 Windows 兼容性
+- 当执行 start.py 时，即使 ZeroNet 已经运行也打开浏览器
+- 在重载时保持插件顺序来允许插件扩展另一个插件
+- 只在完整加载 sites.json 时保存来避免数据丢失
+- 将更多的 Tracker 更改为更可靠的 Tracker
+- 更少的 findhashid CPU 使用率
+- 合并下载大量可选文件
+- 更多对于可选文件的其他优化
+- 如果一个站点有 1000 个节点，更积极地清理
+- 为验证错误使用警告而不是错误
+- 首先推送更新到更新的客户端
+- 损坏文件重置改进
+
+### 修复
+- 修复启动时出现的站点删除错误
+- 延迟 WebSocket 消息直到连接上
+- 修复如果文件包含额外数据时的数据库导入
+- 修复大站点下载
+- 修复 diff 发送 bug （跟踪它好长时间了）
+- 修复当 JSON 文件包含 [] 字符时随机出现的发布错误
+- 修复 siteDelete 与 siteCreate bug
+- 修复文件写入确认对话框
+
+
+## ZeroNet 0.4.1 (2016-09-05)
+### 新增
+- 更快启动与更少内存使用的内核改变
+- 尝试连接丢失时重新连接 Tor
+- 侧边栏滑入
+- 尝试避免不完整的数据文件被覆盖
+- 更快地打开数据库
+- 在侧边栏显示用户文件大小
+- 依赖 --connection_limit 的并发 worker 数量
+
+
+### 更改
+- 在空闲 5 分钟后关闭数据库
+- 更好的站点大小计算
+- 允许在域名中使用“-”符号
+- 总是尝试为站点保持连接
+- 移除已合并站点的合并权限
+- 只扫描最后 3 天的新闻源来加快数据库请求
+- 更新 ZeroBundle-win 到 Python 2.7.12
+
+
+### 修复
+- 修复重要的安全问题：允许任意用户无需有效的来自 ID 提供者的证书发布新内容，感谢 Kaffie 指出
+- 修复在没有选择提供证书提供者时的侧边栏错误
+- 在数据库重建时跳过无效文件
+- 修复随机弹出的 WebSocket 连接错误
+- 修复新的 siteCreate 命令
+- 修复站点大小计算
+- 修复计算机唤醒后的端口打开检查
+- 修复 --size_limit 的命令行解析
+
+
+## ZeroNet 0.4.0 (2016-08-11)
+### 新增
+- 合并站点插件
+- Live source code reloading: Faster core development by allowing me to make changes in ZeroNet source code without restarting it.
+- 为合并站点设计的新 JSON 表
+- 从侧边栏重建数据库
+- 允许直接在 JSON 表中存储自定义数据：更简单与快速的 SQL 查询
+- 用户文件存档：允许站点拥有者存档不活跃的用户内容到单个文件（减少初始同步的时间/CPU/内存使用率）
+- 在文件删除时同时触发数据库 onUpdated/update
+- 从 ZeroFrame API 请求权限
+- 允许使用 fileWrite API 命令在 content.json 存储额外数据
+- 更快的可选文件下载
+- 使用替代源 (Gogs, Gitlab) 来下载更新
+- Track provided sites/connection and prefer to keep the ones with more sites to reduce connection number
+
+### 更改
+- 保持每个站点至少 5 个连接
+- 将目标站点连接从 10 更改到 15
+- ZeroHello 搜索功能稳定性/速度改进
+- 提升机械硬盘下的客户端性能
+
+### 修复
+- 修复 IE11 wrapper nonce 错误
+- 修复在移动设备上的侧边栏
+- 修复站点大小计算
+- 修复 IE10 兼容性
+- Windows XP ZeroBundle 兼容性（感谢中国人民）
+
+
+## ZeroNet 0.3.7 (2016-05-27)
+### 更改
+- 通过只传输补丁来减少带宽使用
+- 其他 CPU /内存优化
+
+
+## ZeroNet 0.3.6 (2016-05-27)
+### 新增
+- 新的 ZeroHello
+- Newsfeed 函数
+
+### 修复
+- 安全性修复
+
+
+## ZeroNet 0.3.5 (2016-02-02)
+### 新增
+- 带有 .onion 隐藏服务的完整 Tor 支持
+- 使用 ZeroNet 协议的 Bootstrap
+
+### 修复
+- 修复 Gevent 1.0.2 兼容性
+
+
+## ZeroNet 0.3.4 (2015-12-28)
+### 新增
+- AES, ECIES API 函数支持
+- PushState 与 ReplaceState URL 通过 API 的操作支持
+- 多用户 localstorage

CHANGELOG.md

@@ -1,201 +1,3 @@
-### ZeroNet 0.9.0 (2023-07-12) Rev4630
- - Fix RDos Issue in Plugins https://github.com/ZeroNetX/ZeroNet-Plugins/pull/9
- - Add trackers to Config.py for failsafety incase missing trackers.txt 
- - Added Proxy links
- - Fix pysha3 dep installation issue
- - FileRequest -> Remove Unnecessary check, Fix error wording
- - Fix Response when site is missing for `actionAs`
-
-
-### ZeroNet 0.8.5 (2023-02-12) Rev4625
- - Fix(https://github.com/ZeroNetX/ZeroNet/pull/202) for SSL cert gen failed on Windows.
- - default theme-class for missing value in `users.json`.
- - Fetch Stats Plugin changes.
-
-### ZeroNet 0.8.4 (2022-12-12) Rev4620
- - Increase Minimum Site size to 25MB.
-
-### ZeroNet 0.8.3 (2022-12-11) Rev4611
- - main.py -> Fix accessing unassigned varible
- - ContentManager -> Support for multiSig
- - SiteStrorage.py -> Fix accessing unassigned varible
- - ContentManager.py Improve Logging of Valid Signers
-
-### ZeroNet 0.8.2 (2022-11-01) Rev4610
- - Fix Startup Error when plugins dir missing
- - Move trackers to seperate file & Add more trackers
- - Config:: Skip loading missing tracker files
- - Added documentation for getRandomPort fn
-
-### ZeroNet 0.8.1 (2022-10-01) Rev4600
- - fix readdress loop (cherry-pick previously added commit from conservancy) 
- - Remove Patreon badge
- - Update README-ru.md (#177) 
- - Include inner_path of failed request for signing in error msg and response
- - Don't Fail Silently When Cert is Not Selected
- - Console Log Updates, Specify min supported ZeroNet version for Rust version Protocol Compatibility
- - Update FUNDING.yml
-
-### ZeroNet 0.8.0 (2022-05-27) Rev4591
- - Revert File Open to catch File Access Errors. 
-
-### ZeroNet 0.7.9-patch (2022-05-26) Rev4586
- - Use xescape(s) from zeronet-conservancy
- - actionUpdate response Optimisation 
- - Fetch Plugins Repo Updates
- - Fix Unhandled File Access Errors 
- - Create codeql-analysis.yml 
-
-### ZeroNet 0.7.9 (2022-05-26) Rev4585
- - Rust Version Compatibility for update Protocol msg 
- - Removed Non Working Trakers.
- - Dynamically Load Trackers from Dashboard Site.
- - Tracker Supply Improvements.
- - Fix Repo Url for Bug Report
- - First Party Tracker Update Service using Dashboard Site.
- - remove old v2 onion service [#158](https://github.com/ZeroNetX/ZeroNet/pull/158)
-
-### ZeroNet 0.7.8 (2022-03-02) Rev4580
- - Update Plugins with some bug fixes and Improvements
-
-### ZeroNet 0.7.6 (2022-01-12) Rev4565
- - Sync Plugin Updates
- - Clean up tor v3 patch [#115](https://github.com/ZeroNetX/ZeroNet/pull/115)
- - Add More Default Plugins to Repo
- - Doubled Site Publish Limits
- - Update ZeroNet Repo Urls [#103](https://github.com/ZeroNetX/ZeroNet/pull/103)
- - UI/UX: Increases Size of Notifications Close Button [#106](https://github.com/ZeroNetX/ZeroNet/pull/106)
- - Moved Plugins to Seperate Repo
- - Added `access_key` variable in Config, this used to access restrited plugins when multiuser plugin is enabled. When MultiUserPlugin is enabled we cannot access some pages like /Stats, this key will remove such restriction with access key.
- - Added `last_connection_id_current_version` to ConnectionServer, helpful to estimate no of connection from current client version.
- - Added current version:  connections to /Stats page. see the previous point.
-
-### ZeroNet 0.7.5 (2021-11-28) Rev4560
- - Add more default trackers
- - Change default homepage address to `1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`
- - Change default update site address to `1Update8crprmciJHwp2WXqkx2c4iYp18`
-
-### ZeroNet 0.7.3 (2021-11-28) Rev4555
- - Fix xrange is undefined error
- - Fix Incorrect viewport on mobile while loading 
- - Tor-V3 Patch by anonymoose
-
-
-### ZeroNet 0.7.1 (2019-07-01) Rev4206
-### Added
- - Built-in logging console in the web UI to see what's happening in the background. (pull down top-right 0 button to see it)
- - Display database rebuild errors [Thanks to Lola]
- - New plugin system that allows to install and manage builtin/third party extensions to the ZeroNet client using the web interface.
- - Support multiple trackers_file
- - Add OpenSSL 1.1 support to CryptMessage plugin based on Bitmessage modifications [Thanks to radfish]
- - Display visual error message on startup errors
- - Fix max opened files changing on Windows platform
- - Display TLS1.3 compatibility on /Stats page
- - Add fake SNI and ALPN to peer connections to make it more like standard https connections
- - Hide and ignore tracker_proxy setting in Tor: Always mode as it's going to use Tor anyway.
- - Deny websocket connections from unknown origins
- - Restrict open_browser values to avoid RCE on sandbox escape
- - Offer access web interface by IP address in case of unknown host
- - Link to site's sidebar with "#ZeroNet:OpenSidebar" hash
-
-### Changed
- - Allow .. in file names [Thanks to imachug]
- - Change unstable trackers
- - More clean errors on sites.json/users.json load error
- - Various tweaks for tracker rating on unstable connections
- - Use OpenSSL 1.1 dlls from default Python Windows distribution if possible
- - Re-factor domain resolving for easier domain plugins
- - Disable UDP connections if --proxy is used
- - New, decorator-based Websocket API permission system to avoid future typo mistakes
-
-### Fixed
- - Fix parsing config lines that have no value
- - Fix start.py [Thanks to imachug]
- - Allow multiple values of the same key in the config file [Thanks ssdifnskdjfnsdjk for reporting]
- - Fix parsing config file lines that has % in the value [Thanks slrslr for reporting]
- - Fix bootstrapper plugin hash reloads [Thanks geekless for reporting]
- - Fix CryptMessage plugin OpenSSL dll loading on Windows (ZeroMail errors) [Thanks cxgreat2014 for reporting]
- - Fix startup error when using OpenSSL 1.1 [Thanks to imachug]
- - Fix a bug that did not loaded merged site data for 5 sec after the merged site got added
- - Fix typo that allowed to add new plugins in public proxy mode. [Thanks styromaniac for reporting]
- - Fix loading non-big files with "|all" postfix [Thanks to krzotr]
- - Fix OpenSSL cert generation error crash by change Windows console encoding to utf8
-
-#### Wrapper html injection vulnerability [Reported by ivanq]
-
-In ZeroNet before rev4188 the wrapper template variables was rendered incorrectly.
-
-Result: The opened site was able to gain WebSocket connection with unrestricted ADMIN/NOSANDBOX access, change configuration values and possible RCE on client's machine.
-
-Fix: Fixed the template rendering code, disallowed WebSocket connections from unknown locations, restricted open_browser configuration values to avoid possible RCE in case of sandbox escape.
-
-Note: The fix is also back ported to ZeroNet Py 2.x version (Rev3870)
-
-
-### ZeroNet 0.7.0 (2019-06-12) Rev4106 (First release targeting Python 3.4+)
-### Added
- - 5-10x faster signature verification by using libsecp256k1 (Thanks to ZeroMux)
- - Generated SSL certificate randomization to avoid protocol filters (Thanks to ValdikSS)
- - Offline mode
- - P2P source code update using ZeroNet protocol
- - ecdsaSign/Verify commands to CryptMessage plugin (Thanks to imachug)
- - Efficient file rename: change file names instead of re-downloading the file.
- - Make redirect optional on site cloning (Thanks to Lola)
- - EccPrivToPub / EccPubToPriv functions (Thanks to imachug)
- - Detect and change dark/light theme based on OS setting (Thanks to filips123)
-
-### Changed
- - Re-factored code to Python3 runtime (compatible with Python 3.4-3.8)
- - More safe database sync mode
- - Removed bundled third-party libraries where it's possible
- - Use lang=en instead of lang={lang} in urls to avoid url encode problems
- - Remove environment details from error page
- - Don't push content.json updates larger than 10kb to significantly reduce bw usage for site with many files
-
-### Fixed
- - Fix sending files with \0 characters
- - Security fix: Escape error detail to avoid XSS (reported by krzotr)
- - Fix signature verification using libsecp256k1 for compressed addresses (mostly certificates generated in the browser)
- - Fix newsfeed if you have more than 1000 followed topic/post on one site.
- - Fix site download as zip file
- - Fix displaying sites with utf8 title
- - Error message if dbRebuild fails (Thanks to Lola)
- - Fix browser reopen if executing start.py again. (Thanks to imachug)
-
-
-### ZeroNet 0.6.5 (2019-02-16) Rev3851 (Last release targeting Python 2.7.x)
-### Added
- - IPv6 support in peer exchange, bigfiles, optional file finding, tracker sharing, socket listening and connecting (based on tangdou1 modifications)
- - New tracker database format with IPv6 support
- - Display notification if there is an unpublished modification for your site
- - Listen and shut down normally for SIGTERM (Thanks to blurHY)
- - Support tilde `~` in filenames (by d14na)
- - Support map for Namecoin subdomain names (Thanks to lola)
- - Add log level to config page
- - Support `{data}` for data dir variable in trackers_file value
- - Quick check content.db on startup and rebuild if necessary
- - Don't show meek proxy option if the tor client does not supports it
-
-### Changed
- - Refactored port open checking with IPv6 support
- - Consider non-local IPs as external even is the open port check fails (for CJDNS and Yggdrasil support)
- - Add IPv6 tracker and change unstable tracker
- - Don't correct sent local time with the calculated time correction
- - Disable CSP for Edge
- - Only support CREATE commands in dbschema indexes node and SELECT from storage.query
-
-### Fixed
- - Check the length of master seed when executing cryptGetPrivatekey CLI command
- - Only reload source code on file modification / creation
- - Detection and issue warning for latest no-script plugin
- - Fix atomic write of a non-existent file
- - Fix sql queries with lots of variables and sites with lots of content.json
- - Fix multi-line parsing of zeronet.conf
- - Fix site deletion from users.json
- - Fix site cloning before site downloaded (Reported by unsystemizer)
- - Fix queryJson for non-list nodes (Reported by MingchenZhang)
-
-
 ## ZeroNet 0.6.4 (2018-10-20) Rev3660
 ### Added
  - New plugin: UiConfig. A web interface that allows changing ZeroNet settings.

COPYING

@@ -1,674 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
-
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU Affero General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.

Dockerfile

@@ -1,33 +1,27 @@
-FROM alpine:3.15 
+FROM alpine:3.8
 
 #Base settings
 ENV HOME /root
 
-COPY requirements.txt /root/requirements.txt
-
-#Install ZeroNet
-RUN apk --update --no-cache --no-progress add python3 python3-dev py3-pip gcc g++ autoconf automake libtool libffi-dev musl-dev make tor openssl \
- && pip3 install -r /root/requirements.txt \
- && apk del python3-dev gcc g++ autoconf automake libtool libffi-dev musl-dev make \
- && echo "ControlPort 9051" >> /etc/tor/torrc \
- && echo "CookieAuthentication 1" >> /etc/tor/torrc
- 
-RUN python3 -V \
- && python3 -m pip list \
- && tor --version \
- && openssl version
-
 #Add Zeronet source
 COPY . /root
+
+#Install ZeroNet
+RUN apk --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \
+ && pip3 install -r /root/requirements.txt \
+ && apk del python3-dev gcc libffi-dev musl-dev make \
+ && echo "ControlPort 9051" >> /etc/tor/torrc \
+ && echo "CookieAuthentication 1" >> /etc/tor/torrc
+
 VOLUME /root/data
 
 #Control if Tor proxy is started
-ENV ENABLE_TOR true
+ENV ENABLE_TOR false
 
 WORKDIR /root
 
 #Set upstart command
-CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26117
+CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
 
 #Expose ports
-EXPOSE 43110 26117
+EXPOSE 43110 26552

Dockerfile.arm64v8

@@ -1,34 +0,0 @@
-FROM alpine:3.12
-
-#Base settings
-ENV HOME /root
-
-COPY requirements.txt /root/requirements.txt
-
-#Install ZeroNet
-RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \
- && pip3 install -r /root/requirements.txt \
- && apk del python3-dev gcc libffi-dev musl-dev make \
- && echo "ControlPort 9051" >> /etc/tor/torrc \
- && echo "CookieAuthentication 1" >> /etc/tor/torrc
- 
-RUN python3 -V \
- && python3 -m pip list \
- && tor --version \
- && openssl version
-
-#Add Zeronet source
-COPY . /root
-VOLUME /root/data
-
-#Control if Tor proxy is started
-ENV ENABLE_TOR false
-
-WORKDIR /root
-
-#Set upstart command
-CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
-
-#Expose ports
-EXPOSE 43110 26552
-

LICENSE

@@ -1,27 +1,340 @@
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-
-Additional Conditions :
-
-Contributing to this repo
- This repo is governed by GPLv3, same is located at the root of the ZeroNet git repo, 
- unless specified separately all code is governed by that license, contributions to this repo 
- are divided into two key types, key contributions and non-key contributions, key contributions 
- are which, directly affects the code performance, quality and features of software, 
- non key contributions include things like translation datasets, image, graphic or video 
- contributions that does not affect the main usability of software but improves the existing 
- usability of certain thing or feature, these also include tests written with code, since their 
- purpose is to check, whether something is working or not as intended. All the non-key contributions 
- are governed by [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/), unless specified 
- above, a contribution is ruled by the type of contribution if there is a conflict between two 
- contributing parties of repo in any case.
+GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    {description}
+    Copyright (C) {year}  {fullname}
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  {signature of Ty Coon}, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
+

README-ru.md

@@ -1,133 +1,211 @@
-# ZeroNet [![tests](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml/badge.svg)](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [![Docker Pulls](https://img.shields.io/docker/pulls/canewsin/zeronet)](https://hub.docker.com/r/canewsin/zeronet)
+# ZeroNet [![Build Status](https://travis-ci.org/HelloZeroNet/ZeroNet.svg?branch=master)](https://travis-ci.org/HelloZeroNet/ZeroNet) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://zeronet.io/docs/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://zeronet.io/docs/help_zeronet/donate/)
 
 [简体中文](./README-zh-cn.md)
 [English](./README.md)
 
-Децентрализованные вебсайты, использующие криптографию Bitcoin и протокол BitTorrent — https://zeronet.dev ([Зеркало в ZeroNet](http://127.0.0.1:43110/1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX/)). В отличии от Bitcoin, ZeroNet'у не требуется блокчейн для работы, однако он использует ту же криптографию, чтобы обеспечить сохранность и проверку данных.
+Децентрализованные вебсайты использующие Bitcoin криптографию и BitTorrent сеть - https://zeronet.io
+
 
 ## Зачем?
 
-- Мы верим в открытую, свободную, и неподдающуюся цензуре сеть и связь.
-- Нет единой точки отказа: Сайт остаётся онлайн, пока его обслуживает хотя бы 1 пир.
-- Нет затрат на хостинг: Сайты обслуживаются посетителями.
-- Невозможно отключить: Он нигде, потому что он везде.
-- Скорость и возможность работать без Интернета: Вы сможете получить доступ к сайту, потому что его копия хранится на вашем компьютере и у ваших пиров.
+* Мы верим в открытую, свободную, и не отцензуренную сеть и коммуникацию.
+* Нет единой точки отказа: Сайт онлайн пока по крайней мере 1 пир обслуживает его.
+* Никаких затрат на хостинг: Сайты обслуживаются посетителями.
+* Невозможно отключить: Он нигде, потому что он везде.
+* Быстр и работает оффлайн: Вы можете получить доступ к сайту, даже если Интернет недоступен.
+
 
 ## Особенности
+ * Обновляемые в реальном времени сайты
+ * Поддержка Namecoin .bit доменов
+ * Лёгок в установке: распаковал & запустил
+ * Клонирование вебсайтов в один клик
+ * Password-less [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
+   based authorization: Ваша учетная запись защищена той же криптографией, что и ваш Bitcoin-кошелек
+ * Встроенный SQL-сервер с синхронизацией данных P2P: Позволяет упростить разработку сайта и ускорить загрузку страницы
+ * Анонимность: Полная поддержка сети Tor с помощью скрытых служб .onion вместо адресов IPv4
+ * TLS зашифрованные связи
+ * Автоматическое открытие uPnP порта
+ * Плагин для поддержки многопользовательской (openproxy)
+ * Работает с любыми браузерами и операционными системами
 
-- Обновление сайтов в реальном времени
-- Поддержка доменов `.bit` ([Namecoin](https://www.namecoin.org))
-- Легкая установка: просто распакуйте и запустите
-- Клонирование сайтов "в один клик"
-- Беспарольная [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
-  авторизация: Ваша учетная запись защищена той же криптографией, что и ваш Bitcoin-кошелек
-- Встроенный SQL-сервер с синхронизацией данных P2P: Позволяет упростить разработку сайта и ускорить загрузку страницы
-- Анонимность: Полная поддержка сети Tor, используя скрытые службы `.onion` вместо адресов IPv4
-- Зашифрованное TLS подключение
-- Автоматическое открытие UPnP–порта
-- Плагин для поддержки нескольких пользователей (openproxy)
-- Работа с любыми браузерами и операционными системами
-
-## Текущие ограничения
-
-- Файловые транзакции не сжаты
-- Нет приватных сайтов
 
 ## Как это работает?
 
-- После запуска `zeronet.py` вы сможете посещать сайты в ZeroNet, используя адрес
-  `http://127.0.0.1:43110/{zeronet_адрес}`
-  (Например: `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`).
-- Когда вы посещаете новый сайт в ZeroNet, он пытается найти пиров с помощью протокола BitTorrent,
-  чтобы скачать у них файлы сайта (HTML, CSS, JS и т.д.).
-- После посещения сайта вы тоже становитесь его пиром.
-- Каждый сайт содержит файл `content.json`, который содержит SHA512 хеши всех остальные файлы
-  и подпись, созданную с помощью закрытого ключа сайта.
-- Если владелец сайта (тот, кто владеет закрытым ключом для адреса сайта) изменяет сайт, он
+* После запуска `zeronet.py` вы сможете посетить зайты (zeronet сайты) используя адрес
+  `http://127.0.0.1:43110/{zeronet_address}`
+(например. `http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D`).
+* Когда вы посещаете новый сайт zeronet, он пытается найти пиров с помощью BitTorrent
+  чтобы загрузить файлы сайтов (html, css, js ...) из них.
+* Каждый посещенный зайт также обслуживается вами. (Т.е хранится у вас на компьютере)
+* Каждый сайт содержит файл `content.json`, который содержит все остальные файлы в хэше sha512
+  и подпись, созданную с использованием частного ключа сайта.
+* Если владелец сайта (у которого есть закрытый ключ для адреса сайта) изменяет сайт, то он/она
   подписывает новый `content.json` и публикует его для пиров. После этого пиры проверяют целостность `content.json`
-  (используя подпись), скачвают изменённые файлы и распространяют новый контент для других пиров.
+  (используя подпись), они загружают измененные файлы и публикуют новый контент для других пиров.
+
+####  [Слайд-шоу о криптографии ZeroNet, обновлениях сайтов, многопользовательских сайтах »](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000)
+####  [Часто задаваемые вопросы »](https://zeronet.io/docs/faq/)
+
+####  [Документация разработчика ZeroNet »](https://zeronet.io/docs/site_development/getting_started/)
 
-[Презентация о криптографии ZeroNet, обновлениях сайтов, многопользовательских сайтах »](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000)
-[Часто задаваемые вопросы »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/)
-[Документация разработчика ZeroNet »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
 
 ## Скриншоты
 
 ![Screenshot](https://i.imgur.com/H60OAHY.png)
 ![ZeroTalk](https://zeronet.io/docs/img/zerotalk.png)
-[Больше скриншотов в документации ZeroNet »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/)
 
-## Как присоединиться?
+#### [Больше скриншотов в ZeroNet документации »](https://zeronet.io/docs/using_zeronet/sample_sites/)
 
-### Windows
 
-- Скачайте и распакуйте архив [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26МБ)
-- Запустите `ZeroNet.exe`
+## Как вступить
 
-### macOS
+* Скачайте ZeroBundle пакет:
+  * [Microsoft Windows](https://github.com/HelloZeroNet/ZeroNet-win/archive/dist/ZeroNet-win.zip)
+  * [Apple macOS](https://github.com/HelloZeroNet/ZeroNet-mac/archive/dist/ZeroNet-mac.zip)
+  * [Linux 64-bit](https://github.com/HelloZeroNet/ZeroBundle/raw/master/dist/ZeroBundle-linux64.tar.gz)
+  * [Linux 32-bit](https://github.com/HelloZeroNet/ZeroBundle/raw/master/dist/ZeroBundle-linux32.tar.gz)
+* Распакуйте где угодно
+* Запустите `ZeroNet.exe` (win), `ZeroNet(.app)` (osx), `ZeroNet.sh` (linux)
 
-- Скачайте и распакуйте архив [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14МБ)
-- Запустите `ZeroNet.app`
+### Linux терминал
 
-### Linux (64 бит)
+* `wget https://github.com/HelloZeroNet/ZeroBundle/raw/master/dist/ZeroBundle-linux64.tar.gz`
+* `tar xvpfz ZeroBundle-linux64.tar.gz`
+* `cd ZeroBundle`
+* Запустите с помощью `./ZeroNet.sh`
 
-- Скачайте и распакуйте архив [ZeroNet-linux.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip) (14МБ)
-- Запустите `./ZeroNet.sh`
+Он загружает последнюю версию ZeroNet, затем запускает её автоматически.
 
-> **Note**
-> Запустите таким образом: `./ZeroNet.sh --ui_ip '*' --ui_restrict ваш_ip_адрес`, чтобы разрешить удалённое подключение к веб–интерфейсу.
+#### Ручная установка для Debian Linux
 
-### Docker
+* `sudo apt-get update`
+* `sudo apt-get install msgpack-python python-gevent`
+* `wget https://github.com/HelloZeroNet/ZeroNet/archive/master.tar.gz`
+* `tar xvpfz master.tar.gz`
+* `cd ZeroNet-master`
+* Запустите с помощью `python2 zeronet.py`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
 
-Официальный образ находится здесь: https://hub.docker.com/r/canewsin/zeronet/
+### [Arch Linux](https://www.archlinux.org)
 
-### Android (arm, arm64, x86)
+* `git clone https://aur.archlinux.org/zeronet.git`
+* `cd zeronet`
+* `makepkg -srci`
+* `systemctl start zeronet`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
 
-- Для работы требуется Android как минимум версии 5.0 Lollipop
-- [<img src="https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png" 
-     alt="Download from Google Play" 
-     height="80">](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile)
-- Скачать APK: https://github.com/canewsin/zeronet_mobile/releases
+Смотрите [ArchWiki](https://wiki.archlinux.org)'s [ZeroNet
+article](https://wiki.archlinux.org/index.php/ZeroNet) для дальнейшей помощи.
 
-### Android (arm, arm64, x86) Облегчённый клиент только для просмотра (1МБ)
+### [Gentoo Linux](https://www.gentoo.org)
 
-- Для работы требуется Android как минимум версии 4.1 Jelly Bean
-- [<img src="https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png" 
-     alt="Download from Google Play" 
-     height="80">](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite)
+* [`layman -a raiagent`](https://github.com/leycec/raiagent)
+* `echo '>=net-vpn/zeronet-0.5.4' >> /etc/portage/package.accept_keywords`
+* *(Опционально)* Включить поддержку Tor: `echo 'net-vpn/zeronet tor' >>
+  /etc/portage/package.use`
+* `emerge zeronet`
+* `rc-service zeronet start`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
 
-### Установка из исходного кода
+Смотрите `/usr/share/doc/zeronet-*/README.gentoo.bz2` для дальнейшей помощи.
 
-```sh
-wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip
-unzip ZeroNet-src.zip
-cd ZeroNet
-sudo apt-get update
-sudo apt-get install python3-pip
-sudo python3 -m pip install -r requirements.txt
+### [FreeBSD](https://www.freebsd.org/)
+
+* `pkg install zeronet` or `cd /usr/ports/security/zeronet/ && make install clean`
+* `sysrc zeronet_enable="YES"`
+* `service zeronet start`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+### [Vagrant](https://www.vagrantup.com/)
+
+* `vagrant up`
+* Подключитесь к VM с помощью `vagrant ssh`
+* `cd /vagrant`
+* Запустите `python2 zeronet.py --ui_ip 0.0.0.0`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+### [Docker](https://www.docker.com/)
+* `docker run -d -v <local_data_folder>:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 nofish/zeronet`
+* Это изображение Docker включает в себя прокси-сервер Tor, который по умолчанию отключён.
+  Остерегайтесь что некоторые хостинг-провайдеры могут не позволить вам запускать Tor на своих серверах.
+  Если вы хотите включить его,установите переменную среды `ENABLE_TOR` в` true` (по умолчанию: `false`) Например:
+
+ `docker run -d -e "ENABLE_TOR=true" -v <local_data_folder>:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 nofish/zeronet`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+### [Virtualenv](https://virtualenv.readthedocs.org/en/latest/)
+
+* `virtualenv env`
+* `source env/bin/activate`
+* `pip install msgpack gevent`
+* `python2 zeronet.py`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+## Текущие ограничения
+
+* ~~Нет torrent-похожего файла разделения для поддержки больших файлов~~ (поддержка больших файлов добавлена)
+* ~~Не анонимнее чем Bittorrent~~ (добавлена встроенная поддержка Tor)
+* Файловые транзакции не сжаты ~~ или незашифрованы еще ~~ (добавлено шифрование TLS)
+* Нет приватных сайтов
+
+
+## Как я могу создать сайт в Zeronet?
+
+Завершите работу zeronet, если он запущен
+
+```bash
+$ zeronet.py siteCreate
+...
+- Site private key (Приватный ключ сайта): 23DKQpzxhbVBrAtvLEc2uvk7DZweh4qL3fn3jpM3LgHDczMK2TtYUq
+- Site address (Адрес сайта): 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+- Site created! (Сайт создан)
+$ zeronet.py
+...
 ```
-- Запустите `python3 zeronet.py`
 
-Откройте приветственную страницу ZeroHello в вашем браузере по ссылке http://127.0.0.1:43110/
+Поздравляем, вы закончили! Теперь каждый может получить доступ к вашему зайту используя
+`http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2`
 
-## Как мне создать сайт в ZeroNet?
+Следующие шаги: [ZeroNet Developer Documentation](https://zeronet.io/docs/site_development/getting_started/)
 
-- Кликните на **⋮** > **"Create new, empty site"** в меню на сайте [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d).
-- Вы будете **перенаправлены** на совершенно новый сайт, который может быть изменён только вами!
-- Вы можете найти и изменить контент вашего сайта в каталоге **data/[адрес_вашего_сайта]**
-- После изменений откройте ваш сайт, переключите влево кнопку "0" в правом верхнем углу, затем нажмите кнопки **sign** и **publish** внизу
 
-Следующие шаги: [Документация разработчика ZeroNet](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
+## Как я могу модифицировать Zeronet сайт?
+
+* Измените файлы расположенные в data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 директории.
+  Когда закончите с изменением:
+
+```bash
+$ zeronet.py siteSign 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+- Signing site (Подпись сайта): 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2...
+Private key (Приватный ключ) (input hidden):
+```
+
+* Введите секретный ключ, который вы получили при создании сайта, потом:
+
+```bash
+$ zeronet.py sitePublish 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+Site:13DNDk..bhC2 Publishing to 3/10 peers...
+Site:13DNDk..bhC2 Successfuly published to 3 peers
+- Serving files....
+```
+
+* Вот и всё! Вы успешно подписали и опубликовали свои изменения.
+
 
 ## Поддержите проект
 
-- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Рекомендуем)
-- LiberaPay: https://liberapay.com/PramUkesh
-- Paypal: https://paypal.me/PramUkesh
-- Другие способы: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive)
+- Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
+- Paypal: https://zeronet.io/docs/help_zeronet/donate/
+
+### Спонсоры
+
+* Улучшенная совместимость с MacOS / Safari стала возможной благодаря [BrowserStack.com](https://www.browserstack.com)
 
 #### Спасибо!
 
-- Здесь вы можете получить больше информации, помощь, прочитать список изменений и исследовать ZeroNet сайты: https://www.reddit.com/r/zeronetx/
-- Общение происходит на канале [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) или в [Gitter](https://gitter.im/canewsin/ZeroNet)
-- Электронная почта: canews.in@gmail.com
+* Больше информации, помощь, журнал изменений, zeronet сайты: https://www.reddit.com/r/zeronet/
+* Приходите, пообщайтесь с нами: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) или на [gitter](https://gitter.im/HelloZeroNet/ZeroNet)
+* Email: hello@zeronet.io (PGP: CB9613AE)

README-zh-cn.md

@@ -1,49 +1,51 @@
-# ZeroNet [![tests](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml/badge.svg)](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [![Docker Pulls](https://img.shields.io/docker/pulls/canewsin/zeronet)](https://hub.docker.com/r/canewsin/zeronet)
+# ZeroNet [![Build Status](https://travis-ci.org/HelloZeroNet/ZeroNet.svg?branch=master)](https://travis-ci.org/HelloZeroNet/ZeroNet) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://zeronet.io/docs/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://zeronet.io/docs/help_zeronet/donate/)
 
 [English](./README.md)
 
-使用 Bitcoin 加密和 BitTorrent 网络的去中心化网络 - https://zeronet.dev
+使用 Bitcoin 加密和 BitTorrent 网络的去中心化网络 - https://zeronet.io
 
 
-## 为什么？
+## 为什么?
 
-* 我们相信开放，自由，无审查的网络和通讯
+* 我们相信开放，自由，无审查的网络
 * 不会受单点故障影响：只要有在线的节点，站点就会保持在线
-* 无托管费用：站点由访问者托管
-* 无法关闭：因为节点无处不在
-* 快速并可离线运行：即使没有互联网连接也可以使用
+* 无托管费用: 站点由访问者托管
+* 无法关闭: 因为节点无处不在
+* 快速并可离线运行: 即使没有互联网连接也可以使用
 
 
 ## 功能
  * 实时站点更新
  * 支持 Namecoin 的 .bit 域名
- * 安装方便：只需解压并运行
+ * 安装方便: 只需解压并运行
  * 一键克隆存在的站点
- * 无需密码、基于 [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
-   的认证：您的账户被与比特币钱包相同的加密方法保护
- * 内建 SQL 服务器和 P2P 数据同步：让开发更简单并提升加载速度
- * 匿名性：完整的 Tor 网络支持，支持通过 .onion 隐藏服务相互连接而不是通过 IPv4 地址连接
+ * 无需密码、基于 [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) 的认证：用与比特币钱包相同的加密方法用来保护你的账户
+你的账户被使用和比特币钱包相同的加密方法
+ * 内建 SQL 服务器和 P2P 数据同步: 让开发更简单并提升加载速度
+ * 匿名性: 完整的 Tor 网络支持，支持通过 .onion 隐藏服务相互连接而不是通过IPv4地址连接
  * TLS 加密连接
  * 自动打开 uPnP 端口
- * 多用户（openproxy）支持的插件
- * 适用于任何浏览器 / 操作系统
+ * 插件和多用户 (开放式代理) 支持
+ * 全平台兼容
 
 
 ## 原理
 
-* 在运行 `zeronet.py` 后，您将可以通过
-  `http://127.0.0.1:43110/{zeronet_address}`（例如：
-  `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`）访问 zeronet 中的站点
-* 在您浏览 zeronet 站点时，客户端会尝试通过 BitTorrent 网络来寻找可用的节点，从而下载需要的文件（html，css，js...）
-* 您将会储存每一个浏览过的站点
-* 每个站点都包含一个名为 `content.json` 的文件，它储存了其他所有文件的 sha512 散列值以及一个通过站点私钥生成的签名
-* 如果站点的所有者（拥有站点地址的私钥）修改了站点，并且他 / 她签名了新的 `content.json` 然后推送至其他节点，
-  那么这些节点将会在使用签名验证 `content.json` 的真实性后，下载修改后的文件并将新内容推送至另外的节点
+* 在你运行`zeronet.py`后你将可以通过`http://127.0.0.1:43110/{zeronet_address}` (比如.
+`http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D`)。访问 zeronet 中的站点。
 
-####  [关于 ZeroNet 加密，站点更新，多用户站点的幻灯片 »](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000)
-####  [常见问题 »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/)
+* 在你浏览 zeronet 站点时，客户端会尝试通过 BitTorrent 网络来寻找可用的节点，从而下载需要的文件 (html, css, js...)
 
-####  [ZeroNet 开发者文档 »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
+* 你将会储存每一个浏览过的站点
+* 每个站点都包含一个名为 `content.json` ，它储存了其他所有文件的 sha512 hash 值
+  和一个通过站点私钥建立的签名
+* 如果站点的所有者 (拥有私钥的那个人) 修改了站点, 并且他/她签名了新的 `content.json` 然后推送至其他节点，
+那么所有节点将会在验证 `content.json` 的真实性 (使用签名)后, 下载修改后的文件并推送至其他节点。
+
+####  [有关于 ZeroNet 加密, 站点更新, 多用户站点的幻灯片 »](https://docs.google.com/presentation/d/1qBxkroB_iiX2zHEn0dt-N-qRZgyEzui46XS2hEa3AA4/pub?start=false&loop=false&delayms=3000)
+####  [常见问题 »](https://zeronet.io/docs/faq/)
+
+####  [ZeroNet开发者文档 »](https://zeronet.io/docs/site_development/getting_started/)
 
 
 ## 屏幕截图
@@ -51,82 +53,136 @@
 ![Screenshot](https://i.imgur.com/H60OAHY.png)
 ![ZeroTalk](https://zeronet.io/docs/img/zerotalk.png)
 
-#### [ZeroNet 文档中的更多屏幕截图 »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/)
+#### [在 ZeroNet 文档里查看更多的屏幕截图 »](https://zeronet.io/docs/using_zeronet/sample_sites/)
 
 
-## 如何加入
+## 如何加入 ？
 
-### Windows
+* 下载 ZeroBundle 文件包:
+  * [Microsoft Windows](https://github.com/HelloZeroNet/ZeroNet-win/archive/dist/ZeroNet-win.zip)
+  * [Apple macOS](https://github.com/HelloZeroNet/ZeroNet-mac/archive/dist/ZeroNet-mac.zip)
+  * [Linux 64bit](https://github.com/HelloZeroNet/ZeroBundle/raw/master/dist/ZeroBundle-linux64.tar.gz)
+  * [Linux 32bit](https://github.com/HelloZeroNet/ZeroBundle/raw/master/dist/ZeroBundle-linux32.tar.gz)
+* 解压缩
+* 运行 `ZeroNet.exe` (win), `ZeroNet(.app)` (osx), `ZeroNet.sh` (linux)
 
- - 下载 [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26MB)
- - 在任意位置解压缩
- - 运行 `ZeroNet.exe`
- 
-### macOS
+### Linux 命令行
 
- - 下载 [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14MB)
- - 在任意位置解压缩
- - 运行 `ZeroNet.app`
- 
-### Linux (x86-64bit)
+* `wget https://github.com/HelloZeroNet/ZeroBundle/raw/master/dist/ZeroBundle-linux64.tar.gz`
+* `tar xvpfz ZeroBundle-linux64.tar.gz`
+* `cd ZeroBundle`
+* 执行 `./ZeroNet.sh` 来启动
 
- - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip`
- - `unzip ZeroNet-linux.zip`
- - `cd ZeroNet-linux`
- - 使用以下命令启动 `./ZeroNet.sh`
- - 在浏览器打开 http://127.0.0.1:43110/ 即可访问 ZeroHello 页面
- 
- __提示：__ 若要允许在 Web 界面上的远程连接，使用以下命令启动 `./ZeroNet.sh --ui_ip '*' --ui_restrict your.ip.address`
+在你打开时他将会自动下载最新版本的 ZeroNet 。
 
-### 从源代码安装
+#### 在 Debian Linux 中手动安装
 
- - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip`
- - `unzip ZeroNet-src.zip`
- - `cd ZeroNet`
- - `sudo apt-get update`
- - `sudo apt-get install python3-pip`
- - `sudo python3 -m pip install -r requirements.txt`
- - 使用以下命令启动 `python3 zeronet.py`
- - 在浏览器打开 http://127.0.0.1:43110/ 即可访问 ZeroHello 页面
+* `sudo apt-get update`
+* `sudo apt-get install msgpack-python python-gevent`
+* `wget https://github.com/HelloZeroNet/ZeroNet/archive/master.tar.gz`
+* `tar xvpfz master.tar.gz`
+* `cd ZeroNet-master`
+* 执行 `python2 zeronet.py` 来启动
+* 在你的浏览器中打开 http://127.0.0.1:43110/
 
- ### Android (arm, arm64, x86)
- - minimum Android version supported 21 (Android 5.0 Lollipop)
- - [<img src="https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png" 
-      alt="Download from Google Play" 
-      height="80">](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile)
- - APK download: https://github.com/canewsin/zeronet_mobile/releases
+### [FreeBSD](https://www.freebsd.org/)
 
-### Android (arm, arm64, x86) Thin Client for Preview Only (Size 1MB)
- - minimum Android version supported 16 (JellyBean)
- - [<img src="https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png" 
-      alt="Download from Google Play" 
-      height="80">](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite)
+* `pkg install zeronet` 或者 `cd /usr/ports/security/zeronet/ && make install clean`
+* `sysrc zeronet_enable="YES"`
+* `service zeronet start`
+* 在你的浏览器中打开 http://127.0.0.1:43110/
+
+### [Vagrant](https://www.vagrantup.com/)
+
+* `vagrant up`
+* 通过 `vagrant ssh` 连接到 VM
+* `cd /vagrant`
+* 运行 `python2 zeronet.py --ui_ip 0.0.0.0`
+* 在你的浏览器中打开 http://127.0.0.1:43110/
+
+### [Docker](https://www.docker.com/)
+* `docker run -d -v <local_data_folder>:/root/data -p 26552:26552 -p 43110:43110 nofish/zeronet`
+* 这个 Docker 镜像包含了 Tor ，但默认是禁用的，因为一些托管商不允许你在他们的服务器上运行 Tor。如果你希望启用它，
+设置 `ENABLE_TOR` 环境变量为 `true` (默认: `false`). E.g.:
+
+ `docker run -d -e "ENABLE_TOR=true" -v <local_data_folder>:/root/data -p 26552:26552 -p 43110:43110 nofish/zeronet`
+* 在你的浏览器中打开 http://127.0.0.1:43110/
+
+### [Virtualenv](https://virtualenv.readthedocs.org/en/latest/)
+
+* `virtualenv env`
+* `source env/bin/activate`
+* `pip install msgpack gevent`
+* `python2 zeronet.py`
+* 在你的浏览器中打开 http://127.0.0.1:43110/
 
 ## 现有限制
 
-* 传输文件时没有压缩
+* ~~没有类似于 BitTorrent 的文件拆分来支持大文件~~ (已添加大文件支持)
+* ~~没有比 BitTorrent 更好的匿名性~~ (已添加内置的完整 Tor 支持)
+* 传输文件时没有压缩~~和加密~~ (已添加 TLS 支持)
 * 不支持私有站点
 
 
-## 如何创建一个 ZeroNet 站点？
+## 如何创建一个 ZeroNet 站点?
 
- * 点击 [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d) 站点的 **⋮** > **「新建空站点」** 菜单项
- * 您将被**重定向**到一个全新的站点，该站点只能由您修改
- * 您可以在 **data/[您的站点地址]** 目录中找到并修改网站的内容
- * 修改后打开您的网站，将右上角的「0」按钮拖到左侧，然后点击底部的**签名**并**发布**按钮
 
-接下来的步骤：[ZeroNet 开发者文档](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
+如果 zeronet 在运行，把它关掉
+执行：
+```bash
+$ zeronet.py siteCreate
+...
+- Site private key: 23DKQpzxhbVBrAtvLEc2uvk7DZweh4qL3fn3jpM3LgHDczMK2TtYUq
+- Site address: 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+- Site created!
+$ zeronet.py
+...
+```
+
+你已经完成了！ 现在任何人都可以通过
+`http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2`
+来访问你的站点
+
+下一步: [ZeroNet 开发者文档](https://zeronet.io/docs/site_development/getting_started/)
+
+
+## 我要如何修改 ZeroNet 站点?
+
+* 修改位于 data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 的目录.
+  在你改好之后:
+
+```bash
+$ zeronet.py siteSign 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+- Signing site: 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2...
+Private key (input hidden):
+```
+
+* 输入你在创建站点时获得的私钥
+
+```bash
+$ zeronet.py sitePublish 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+Site:13DNDk..bhC2 Publishing to 3/10 peers...
+Site:13DNDk..bhC2 Successfuly published to 3 peers
+- Serving files....
+```
+
+* 就是这样! 你现在已经成功的签名并推送了你的更改。
+
 
 ## 帮助这个项目
-- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Preferred)
-- LiberaPay: https://liberapay.com/PramUkesh
-- Paypal: https://paypal.me/PramUkesh
-- Others: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive)
 
+- Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
+- Paypal: https://zeronet.io/docs/help_zeronet/donate/
 
-#### 感谢您！
+### 赞助商
 
-* 更多信息，帮助，变更记录和 zeronet 站点：https://www.reddit.com/r/zeronetx/
-* 前往 [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) 或 [gitter](https://gitter.im/canewsin/ZeroNet) 和我们聊天
-* [这里](https://gitter.im/canewsin/ZeroNet)是一个 gitter 上的中文聊天室
-* Email: canews.in@gmail.com
+* 在 OSX/Safari 下 [BrowserStack.com](https://www.browserstack.com) 带来更好的兼容性
+
+#### 感谢!
+
+* 更多信息, 帮助, 变更记录和 zeronet 站点: https://www.reddit.com/r/zeronet/
+* 在: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) 和我们聊天，或者使用 [gitter](https://gitter.im/HelloZeroNet/ZeroNet)
+* [这里](https://gitter.im/ZeroNet-zh/Lobby)是一个 gitter 上的中文聊天室
+* Email: hello@noloop.me

README.md

@@ -1,6 +1,8 @@
-# ZeroNet [![tests](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml/badge.svg)](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [![Docker Pulls](https://img.shields.io/docker/pulls/canewsin/zeronet)](https://hub.docker.com/r/canewsin/zeronet)
-<!--TODO: Update Onion Site -->
-Decentralized websites using Bitcoin crypto and the BitTorrent network - https://zeronet.dev / [ZeroNet Site](http://127.0.0.1:43110/1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX/), Unlike Bitcoin, ZeroNet Doesn't need a blockchain to run, But uses cryptography used by BTC, to ensure data integrity and validation.
+__Warning: Development test version, do not use on live data__
+
+# ZeroNet [![Build Status](https://travis-ci.org/HelloZeroNet/ZeroNet.svg?branch=master)](https://travis-ci.org/HelloZeroNet/ZeroNet) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://zeronet.io/docs/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://zeronet.io/docs/help_zeronet/donate/)
+
+Decentralized websites using Bitcoin crypto and the BitTorrent network - https://zeronet.io
 
 
 ## Why?
@@ -33,22 +35,22 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/
 
 * After starting `zeronet.py` you will be able to visit zeronet sites using
   `http://127.0.0.1:43110/{zeronet_address}` (eg.
-  `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`).
+  `http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D`).
 * When you visit a new zeronet site, it tries to find peers using the BitTorrent
   network so it can download the site files (html, css, js...) from them.
 * Each visited site is also served by you.
 * Every site contains a `content.json` file which holds all other files in a sha512 hash
   and a signature generated using the site's private key.
 * If the site owner (who has the private key for the site address) modifies the
-  site and signs the new `content.json` and publishes it to the peers.
+  site, then he/she signs the new `content.json` and publishes it to the peers.
   Afterwards, the peers verify the `content.json` integrity (using the
   signature), they download the modified files and publish the new content to
   other peers.
 
 ####  [Slideshow about ZeroNet cryptography, site updates, multi-user sites »](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000)
-####  [Frequently asked questions »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/)
+####  [Frequently asked questions »](https://zeronet.io/docs/faq/)
 
-####  [ZeroNet Developer Documentation »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
+####  [ZeroNet Developer Documentation »](https://zeronet.io/docs/site_development/getting_started/)
 
 
 ## Screenshots
@@ -56,101 +58,116 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/
 ![Screenshot](https://i.imgur.com/H60OAHY.png)
 ![ZeroTalk](https://zeronet.io/docs/img/zerotalk.png)
 
-#### [More screenshots in ZeroNet docs »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/)
+#### [More screenshots in ZeroNet docs »](https://zeronet.io/docs/using_zeronet/sample_sites/)
 
 
 ## How to join
 
-### Windows
-
- - Download [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26MB)
- - Unpack anywhere
- - Run `ZeroNet.exe`
- 
-### macOS
-
- - Download [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14MB)
- - Unpack anywhere
- - Run `ZeroNet.app`
- 
-### Linux (x86-64bit)
- - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip`
- - `unzip ZeroNet-linux.zip`
- - `cd ZeroNet-linux`
- - Start with: `./ZeroNet.sh`
- - Open the ZeroHello landing page in your browser by navigating to: http://127.0.0.1:43110/
- 
- __Tip:__ Start with `./ZeroNet.sh --ui_ip '*' --ui_restrict your.ip.address` to allow remote connections on the web interface.
- 
- ### Android (arm, arm64, x86)
- - minimum Android version supported 21 (Android 5.0 Lollipop)
- - [<img src="https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png" 
-      alt="Download from Google Play" 
-      height="80">](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile)
- - APK download: https://github.com/canewsin/zeronet_mobile/releases
-
-### Android (arm, arm64, x86) Thin Client for Preview Only (Size 1MB)
- - minimum Android version supported 16 (JellyBean)
- - [<img src="https://play.google.com/intl/en_us/badges/images/generic/en_badge_web_generic.png" 
-      alt="Download from Google Play" 
-      height="80">](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite)
-
-
-#### Docker
-There is an official image, built from source at: https://hub.docker.com/r/canewsin/zeronet/
-
-### Online Proxies
-Proxies are like seed boxes for sites(i.e ZNX runs on a cloud vps), you can try zeronet experience from proxies. Add your proxy below if you have one.
-
-#### Official ZNX Proxy : 
-
-https://proxy.zeronet.dev/
-
-https://zeronet.dev/
-
-#### From Community
-
-https://0net-preview.com/
-
-https://portal.ngnoid.tv/
-
-https://zeronet.ipfsscan.io/
+### Install from package for your distribution
 
+* Arch Linux: [zeronet](https://aur.archlinux.org/zeronet.git), [zeronet-git](https://aur.archlinux.org/zeronet-git.git)
+* Gentoo:  [emerge repository](https://github.com/leycec/raiagent)
+* FreeBSD: zeronet
+* Whonix: [instructions](https://www.whonix.org/wiki/ZeroNet)
 
 ### Install from source
 
- - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip`
- - `unzip ZeroNet-src.zip`
- - `cd ZeroNet`
- - `sudo apt-get update`
- - `sudo apt-get install python3-pip`
- - `sudo python3 -m pip install -r requirements.txt`
- - Start with: `python3 zeronet.py`
- - Open the ZeroHello landing page in your browser by navigating to: http://127.0.0.1:43110/
+Fetch and extract the source:
+
+    wget https://github.com/HelloZeroNet/ZeroNet/archive/py3.tar.gz
+    tar xvpfz py3.tar.gz
+    cd ZeroNet-py3
+
+Install Python module dependencies either:
+
+* (Option A) into a [virtual env](https://virtualenv.readthedocs.org/en/latest/)
+
+    ```
+    virtualenv zeronet
+    source zeronet/bin/activate
+    python -m pip install -r requirements.txt
+    ```
+
+* (Option B) into the system (requires root), for example, on Debian/Ubuntu:
+
+    ```
+    sudo apt-get update
+    sudo apt-get install python3-pip
+    sudo python3 -m pip install -r requirements.txt
+    ```
+
+Start Zeronet:
+
+    python3 zeronet.py
+
+Open the ZeroHello landing page in your browser by navigating to:
+
+    http://127.0.0.1:43110/
 
 ## Current limitations
 
-* File transactions are not compressed
+* ~~No torrent-like file splitting for big file support~~ (big file support added)
+* ~~No more anonymous than Bittorrent~~ (built-in full Tor support added)
+* File transactions are not compressed ~~or encrypted yet~~ (TLS encryption added)
 * No private sites
 
 
 ## How can I create a ZeroNet site?
 
- * Click on **⋮** > **"Create new, empty site"** menu item on the site [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d).
- * You will be **redirected** to a completely new site that is only modifiable by you!
- * You can find and modify your site's content in **data/[yoursiteaddress]** directory
- * After the modifications open your site, drag the topright "0" button to left, then press **sign** and **publish** buttons on the bottom
+Shut down zeronet if you are running it already
+
+```bash
+$ zeronet.py siteCreate
+...
+- Site private key: 23DKQpzxhbVBrAtvLEc2uvk7DZweh4qL3fn3jpM3LgHDczMK2TtYUq
+- Site address: 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+- Site created!
+$ zeronet.py
+...
+```
+
+Congratulations, you're finished! Now anyone can access your site using
+`http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2`
+
+Next steps: [ZeroNet Developer Documentation](https://zeronet.io/docs/site_development/getting_started/)
+
+
+## How can I modify a ZeroNet site?
+
+* Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory.
+  After you're finished:
+
+```bash
+$ zeronet.py siteSign 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+- Signing site: 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2...
+Private key (input hidden):
+```
+
+* Enter the private key you got when you created the site, then:
+
+```bash
+$ zeronet.py sitePublish 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+Site:13DNDk..bhC2 Publishing to 3/10 peers...
+Site:13DNDk..bhC2 Successfuly published to 3 peers
+- Serving files....
+```
+
+* That's it! You've successfully signed and published your modifications.
 
-Next steps: [ZeroNet Developer Documentation](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
 
 ## Help keep this project alive
-- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Preferred)
-- LiberaPay: https://liberapay.com/PramUkesh
-- Paypal: https://paypal.me/PramUkesh
-- Others: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive)
+
+- Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
+- Paypal: https://zeronet.io/docs/help_zeronet/donate/
+
+### Sponsors
+
+* Better macOS/Safari compatibility made possible by [BrowserStack.com](https://www.browserstack.com)
 
 #### Thank you!
 
-* More info, help, changelog, zeronet sites: https://www.reddit.com/r/zeronetx/
-* Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) or on [gitter](https://gitter.im/canewsin/ZeroNet)
-* Email: canews.in@gmail.com
+* More info, help, changelog, zeronet sites: https://www.reddit.com/r/zeronet/
+* Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) or on [gitter](https://gitter.im/HelloZeroNet/ZeroNet)
+* Email: hello@zeronet.io (PGP: CB9613AE)

plugins

@@ -1 +0,0 @@
-Subproject commit 689d9309f73371f4681191b125ec3f2e14075eeb

plugins/AnnounceLocal/AnnounceLocalPlugin.py

@@ -0,0 +1,148 @@
+import time
+
+import gevent
+
+from Plugin import PluginManager
+from Config import config
+from . import BroadcastServer
+
+
+@PluginManager.registerTo("SiteAnnouncer")
+class SiteAnnouncerPlugin(object):
+    def announce(self, force=False, *args, **kwargs):
+        local_announcer = self.site.connection_server.local_announcer
+
+        thread = None
+        if local_announcer and (force or time.time() - local_announcer.last_discover > 5 * 60):
+            thread = gevent.spawn(local_announcer.discover, force=force)
+        back = super(SiteAnnouncerPlugin, self).announce(force=force, *args, **kwargs)
+
+        if thread:
+            thread.join()
+
+        return back
+
+
+class LocalAnnouncer(BroadcastServer.BroadcastServer):
+    def __init__(self, server, listen_port):
+        super(LocalAnnouncer, self).__init__("zeronet", listen_port=listen_port)
+        self.server = server
+
+        self.sender_info["peer_id"] = self.server.peer_id
+        self.sender_info["port"] = self.server.port
+        self.sender_info["broadcast_port"] = listen_port
+        self.sender_info["rev"] = config.rev
+
+        self.known_peers = {}
+        self.last_discover = 0
+
+    def discover(self, force=False):
+        self.log.debug("Sending discover request (force: %s)" % force)
+        self.last_discover = time.time()
+        if force:  # Probably new site added, clean cache
+            self.known_peers = {}
+
+        for peer_id, known_peer in list(self.known_peers.items()):
+            if time.time() - known_peer["found"] > 20 * 60:
+                del(self.known_peers[peer_id])
+                self.log.debug("Timeout, removing from known_peers: %s" % peer_id)
+        self.broadcast({"cmd": "discoverRequest", "params": {}}, port=self.listen_port)
+
+    def actionDiscoverRequest(self, sender, params):
+        back = {
+            "cmd": "discoverResponse",
+            "params": {
+                "sites_changed": self.server.site_manager.sites_changed
+            }
+        }
+
+        if sender["peer_id"] not in self.known_peers:
+            self.known_peers[sender["peer_id"]] = {"added": time.time(), "sites_changed": 0, "updated": 0, "found": time.time()}
+            self.log.debug("Got discover request from unknown peer %s (%s), time to refresh known peers" % (sender["ip"], sender["peer_id"]))
+            gevent.spawn_later(1.0, self.discover)  # Let the response arrive first to the requester
+
+        return back
+
+    def actionDiscoverResponse(self, sender, params):
+        if sender["peer_id"] in self.known_peers:
+            self.known_peers[sender["peer_id"]]["found"] = time.time()
+        if params["sites_changed"] != self.known_peers.get(sender["peer_id"], {}).get("sites_changed"):
+            # Peer's site list changed, request the list of new sites
+            return {"cmd": "siteListRequest"}
+        else:
+            # Peer's site list is the same
+            for site in self.server.sites.values():
+                peer = site.peers.get("%s:%s" % (sender["ip"], sender["port"]))
+                if peer:
+                    peer.found("local")
+
+    def actionSiteListRequest(self, sender, params):
+        back = []
+        sites = list(self.server.sites.values())
+
+        # Split adresses to group of 100 to avoid UDP size limit
+        site_groups = [sites[i:i + 100] for i in range(0, len(sites), 100)]
+        for site_group in site_groups:
+            res = {}
+            res["sites_changed"] = self.server.site_manager.sites_changed
+            res["sites"] = [site.address_hash for site in site_group]
+            back.append({"cmd": "siteListResponse", "params": res})
+        return back
+
+    def actionSiteListResponse(self, sender, params):
+        s = time.time()
+        peer_sites = set(params["sites"])
+        num_found = 0
+        added_sites = []
+        for site in self.server.sites.values():
+            if site.address_hash in peer_sites:
+                added = site.addPeer(sender["ip"], sender["port"], source="local")
+                num_found += 1
+                if added:
+                    site.worker_manager.onPeers()
+                    site.updateWebsocket(peers_added=1)
+                    added_sites.append(site)
+
+        # Save sites changed value to avoid unnecessary site list download
+        if sender["peer_id"] not in self.known_peers:
+            self.known_peers[sender["peer_id"]] = {"added": time.time()}
+
+        self.known_peers[sender["peer_id"]]["sites_changed"] = params["sites_changed"]
+        self.known_peers[sender["peer_id"]]["updated"] = time.time()
+        self.known_peers[sender["peer_id"]]["found"] = time.time()
+
+        self.log.debug(
+            "Tracker result: Discover from %s response parsed in %.3fs, found: %s added: %s of %s" %
+            (sender["ip"], time.time() - s, num_found, added_sites, len(peer_sites))
+        )
+
+
+@PluginManager.registerTo("FileServer")
+class FileServerPlugin(object):
+    def __init__(self, *args, **kwargs):
+        res = super(FileServerPlugin, self).__init__(*args, **kwargs)
+        if config.broadcast_port and config.tor != "always" and not config.disable_udp:
+            self.local_announcer = LocalAnnouncer(self, config.broadcast_port)
+        else:
+            self.local_announcer = None
+        return res
+
+    def start(self, *args, **kwargs):
+        if self.local_announcer:
+            gevent.spawn(self.local_announcer.start)
+        return super(FileServerPlugin, self).start(*args, **kwargs)
+
+    def stop(self):
+        if self.local_announcer:
+            self.local_announcer.stop()
+        res = super(FileServerPlugin, self).stop()
+        return res
+
+
+@PluginManager.registerTo("ConfigPlugin")
+class ConfigPlugin(object):
+    def createArguments(self):
+        group = self.parser.add_argument_group("AnnounceLocal plugin")
+        group.add_argument('--broadcast_port', help='UDP broadcasting port for local peer discovery', default=1544, type=int, metavar='port')
+
+        return super(ConfigPlugin, self).createArguments()

plugins/AnnounceLocal/BroadcastServer.py

@@ -0,0 +1,139 @@
+import socket
+import logging
+import time
+from contextlib import closing
+
+from Debug import Debug
+from util import UpnpPunch
+from util import Msgpack
+
+
+class BroadcastServer(object):
+    def __init__(self, service_name, listen_port=1544, listen_ip=''):
+        self.log = logging.getLogger("BroadcastServer")
+        self.listen_port = listen_port
+        self.listen_ip = listen_ip
+
+        self.running = False
+        self.sock = None
+        self.sender_info = {"service": service_name}
+
+    def createBroadcastSocket(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        if hasattr(socket, 'SO_REUSEPORT'):
+            try:
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+            except Exception as err:
+                self.log.warning("Error setting SO_REUSEPORT: %s" % err)
+
+        binded = False
+        for retry in range(3):
+            try:
+                sock.bind((self.listen_ip, self.listen_port))
+                binded = True
+                break
+            except Exception as err:
+                self.log.error(
+                    "Socket bind to %s:%s error: %s, retry #%s" %
+                    (self.listen_ip, self.listen_port, Debug.formatException(err), retry)
+                )
+                time.sleep(retry)
+
+        if binded:
+            return sock
+        else:
+            return False
+
+    def start(self):  # Listens for discover requests
+        self.sock = self.createBroadcastSocket()
+        if not self.sock:
+            self.log.error("Unable to listen on port %s" % self.listen_port)
+            return
+
+        self.log.debug("Started on port %s" % self.listen_port)
+
+        self.running = True
+
+        while self.running:
+            try:
+                data, addr = self.sock.recvfrom(8192)
+            except Exception as err:
+                if self.running:
+                    self.log.error("Listener receive error: %s" % err)
+                continue
+
+            if not self.running:
+                break
+
+            try:
+                message = Msgpack.unpack(data)
+                response_addr, message = self.handleMessage(addr, message)
+                if message:
+                    self.send(response_addr, message)
+            except Exception as err:
+                self.log.error("Handlemessage error: %s" % Debug.formatException(err))
+        self.log.debug("Stopped listening on port %s" % self.listen_port)
+
+    def stop(self):
+        self.log.debug("Stopping, socket: %s" % self.sock)
+        self.running = False
+        if self.sock:
+            self.sock.close()
+
+    def send(self, addr, message):
+        if type(message) is not list:
+            message = [message]
+
+        for message_part in message:
+            message_part["sender"] = self.sender_info
+
+            self.log.debug("Send to %s: %s" % (addr, message_part["cmd"]))
+            with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
+                sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+                sock.sendto(Msgpack.pack(message_part), addr)
+
+    def getMyIps(self):
+        return UpnpPunch._get_local_ips()
+
+    def broadcast(self, message, port=None):
+        if not port:
+            port = self.listen_port
+
+        my_ips = self.getMyIps()
+        addr = ("255.255.255.255", port)
+
+        message["sender"] = self.sender_info
+        self.log.debug("Broadcast using ips %s on port %s: %s" % (my_ips, port, message["cmd"]))
+
+        for my_ip in my_ips:
+            try:
+                with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
+                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+                    sock.bind((my_ip, 0))
+                    sock.sendto(Msgpack.pack(message), addr)
+            except Exception as err:
+                self.log.warning("Error sending broadcast using ip %s: %s" % (my_ip, err))
+
+    def handleMessage(self, addr, message):
+        self.log.debug("Got from %s: %s" % (addr, message["cmd"]))
+        cmd = message["cmd"]
+        params = message.get("params", {})
+        sender = message["sender"]
+        sender["ip"] = addr[0]
+
+        func_name = "action" + cmd[0].upper() + cmd[1:]
+        func = getattr(self, func_name, None)
+
+        if sender["service"] != "zeronet" or sender["peer_id"] == self.sender_info["peer_id"]:
+            # Skip messages not for us or sent by us
+            message = None
+        elif func:
+            message = func(sender, params)
+        else:
+            self.log.debug("Unknown cmd: %s" % cmd)
+            message = None
+
+        return (sender["ip"], sender["broadcast_port"]), message

plugins/AnnounceLocal/Test/TestAnnounce.py

@@ -0,0 +1,113 @@
+import time
+import copy
+
+import gevent
+import pytest
+import mock
+
+from AnnounceLocal import AnnounceLocalPlugin
+from File import FileServer
+from Test import Spy
+
+@pytest.fixture
+def announcer(file_server, site):
+    file_server.sites[site.address] = site
+    announcer = AnnounceLocalPlugin.LocalAnnouncer(file_server, listen_port=1100)
+    file_server.local_announcer = announcer
+    announcer.listen_port = 1100
+    announcer.sender_info["broadcast_port"] = 1100
+    announcer.getMyIps = mock.MagicMock(return_value=["127.0.0.1"])
+    announcer.discover = mock.MagicMock(return_value=False)  # Don't send discover requests automatically
+    gevent.spawn(announcer.start)
+    time.sleep(0.5)
+
+    assert file_server.local_announcer.running
+    return file_server.local_announcer
+
+@pytest.fixture
+def announcer_remote(request, site_temp):
+    file_server_remote = FileServer("127.0.0.1", 1545)
+    file_server_remote.sites[site_temp.address] = site_temp
+    announcer = AnnounceLocalPlugin.LocalAnnouncer(file_server_remote, listen_port=1101)
+    file_server_remote.local_announcer = announcer
+    announcer.listen_port = 1101
+    announcer.sender_info["broadcast_port"] = 1101
+    announcer.getMyIps = mock.MagicMock(return_value=["127.0.0.1"])
+    announcer.discover = mock.MagicMock(return_value=False)  # Don't send discover requests automatically
+    gevent.spawn(announcer.start)
+    time.sleep(0.5)
+
+    assert file_server_remote.local_announcer.running
+
+    def cleanup():
+        file_server_remote.stop()
+    request.addfinalizer(cleanup)
+
+
+    return file_server_remote.local_announcer
+
+@pytest.mark.usefixtures("resetSettings")
+@pytest.mark.usefixtures("resetTempSettings")
+class TestAnnounce:
+    def testSenderInfo(self, announcer):
+        sender_info = announcer.sender_info
+        assert sender_info["port"] > 0
+        assert len(sender_info["peer_id"]) == 20
+        assert sender_info["rev"] > 0
+
+    def testIgnoreSelfMessages(self, announcer):
+        # No response to messages that has same peer_id as server
+        assert not announcer.handleMessage(("0.0.0.0", 123), {"cmd": "discoverRequest", "sender": announcer.sender_info, "params": {}})[1]
+
+        # Response to messages with different peer id
+        sender_info = copy.copy(announcer.sender_info)
+        sender_info["peer_id"] += "-"
+        addr, res = announcer.handleMessage(("0.0.0.0", 123), {"cmd": "discoverRequest", "sender": sender_info, "params": {}})
+        assert res["params"]["sites_changed"] > 0
+
+    def testDiscoverRequest(self, announcer, announcer_remote):
+        assert len(announcer_remote.known_peers) == 0
+        with Spy.Spy(announcer_remote, "handleMessage") as responses:
+            announcer_remote.broadcast({"cmd": "discoverRequest", "params": {}}, port=announcer.listen_port)
+            time.sleep(0.1)
+
+        response_cmds = [response[1]["cmd"] for response in responses]
+        assert response_cmds == ["discoverResponse", "siteListResponse"]
+        assert len(responses[-1][1]["params"]["sites"]) == 1
+
+        # It should only request siteList if sites_changed value is different from last response
+        with Spy.Spy(announcer_remote, "handleMessage") as responses:
+            announcer_remote.broadcast({"cmd": "discoverRequest", "params": {}}, port=announcer.listen_port)
+            time.sleep(0.1)
+
+        response_cmds = [response[1]["cmd"] for response in responses]
+        assert response_cmds == ["discoverResponse"]
+
+    def testPeerDiscover(self, announcer, announcer_remote, site):
+        assert announcer.server.peer_id != announcer_remote.server.peer_id
+        assert len(list(announcer.server.sites.values())[0].peers) == 0
+        announcer.broadcast({"cmd": "discoverRequest"}, port=announcer_remote.listen_port)
+        time.sleep(0.1)
+        assert len(list(announcer.server.sites.values())[0].peers) == 1
+
+    def testRecentPeerList(self, announcer, announcer_remote, site):
+        assert len(site.peers_recent) == 0
+        assert len(site.peers) == 0
+        with Spy.Spy(announcer, "handleMessage") as responses:
+            announcer.broadcast({"cmd": "discoverRequest", "params": {}}, port=announcer_remote.listen_port)
+            time.sleep(0.1)
+        assert [response[1]["cmd"] for response in responses] == ["discoverResponse", "siteListResponse"]
+        assert len(site.peers_recent) == 1
+        assert len(site.peers) == 1
+
+        # It should update peer without siteListResponse
+        last_time_found = list(site.peers.values())[0].time_found
+        site.peers_recent.clear()
+        with Spy.Spy(announcer, "handleMessage") as responses:
+            announcer.broadcast({"cmd": "discoverRequest", "params": {}}, port=announcer_remote.listen_port)
+            time.sleep(0.1)
+        assert [response[1]["cmd"] for response in responses] == ["discoverResponse"]
+        assert len(site.peers_recent) == 1
+        assert list(site.peers.values())[0].time_found > last_time_found
+
+

plugins/AnnounceLocal/Test/conftest.py

@@ -0,0 +1,4 @@
+from src.Test.conftest import *
+
+from Config import config
+config.broadcast_port = 0

plugins/AnnounceLocal/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/AnnounceLocal/__init__.py

@@ -0,0 +1 @@
+from . import AnnounceLocalPlugin

plugins/AnnounceShare/AnnounceSharePlugin.py

@@ -0,0 +1,190 @@
+import time
+import os
+import logging
+import json
+import atexit
+
+import gevent
+
+from Config import config
+from Plugin import PluginManager
+from util import helper
+
+
+class TrackerStorage(object):
+    def __init__(self):
+        self.log = logging.getLogger("TrackerStorage")
+        self.file_path = "%s/trackers.json" % config.data_dir
+        self.load()
+        self.time_discover = 0.0
+        atexit.register(self.save)
+
+    def getDefaultFile(self):
+        return {"shared": {}}
+
+    def onTrackerFound(self, tracker_address, type="shared", my=False):
+        if not tracker_address.startswith("zero://"):
+            return False
+
+        trackers = self.getTrackers()
+        added = False
+        if tracker_address not in trackers:
+            trackers[tracker_address] = {
+                "time_added": time.time(),
+                "time_success": 0,
+                "latency": 99.0,
+                "num_error": 0,
+                "my": False
+            }
+            self.log.debug("New tracker found: %s" % tracker_address)
+            added = True
+
+        trackers[tracker_address]["time_found"] = time.time()
+        trackers[tracker_address]["my"] = my
+        return added
+
+    def onTrackerSuccess(self, tracker_address, latency):
+        trackers = self.getTrackers()
+        if tracker_address not in trackers:
+            return False
+
+        trackers[tracker_address]["latency"] = latency
+        trackers[tracker_address]["time_success"] = time.time()
+        trackers[tracker_address]["num_error"] = 0
+
+    def onTrackerError(self, tracker_address):
+        trackers = self.getTrackers()
+        if tracker_address not in trackers:
+            return False
+
+        trackers[tracker_address]["time_error"] = time.time()
+        trackers[tracker_address]["num_error"] += 1
+
+        if len(self.getWorkingTrackers()) >= config.working_shared_trackers_limit:
+            error_limit = 5
+        else:
+            error_limit = 30
+        error_limit
+
+        if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - 60 * 60:
+            self.log.debug("Tracker %s looks down, removing." % tracker_address)
+            del trackers[tracker_address]
+
+    def getTrackers(self, type="shared"):
+        return self.file_content.setdefault(type, {})
+
+    def getWorkingTrackers(self, type="shared"):
+        trackers = {
+            key: tracker for key, tracker in self.getTrackers(type).items()
+            if tracker["time_success"] > time.time() - 60 * 60
+        }
+        return trackers
+
+    def getFileContent(self):
+        if not os.path.isfile(self.file_path):
+            open(self.file_path, "w").write("{}")
+            return self.getDefaultFile()
+        try:
+            return json.load(open(self.file_path))
+        except Exception as err:
+            self.log.error("Error loading trackers list: %s" % err)
+            return self.getDefaultFile()
+
+    def load(self):
+        self.file_content = self.getFileContent()
+
+        trackers = self.getTrackers()
+        self.log.debug("Loaded %s shared trackers" % len(trackers))
+        for address, tracker in list(trackers.items()):
+            tracker["num_error"] = 0
+            if not address.startswith("zero://"):
+                del trackers[address]
+
+    def save(self):
+        s = time.time()
+        helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8"))
+        self.log.debug("Saved in %.3fs" % (time.time() - s))
+
+    def discoverTrackers(self, peers):
+        if len(self.getWorkingTrackers()) > config.working_shared_trackers_limit:
+            return False
+        s = time.time()
+        num_success = 0
+        for peer in peers:
+            if peer.connection and peer.connection.handshake.get("rev", 0) < 3560:
+                continue  # Not supported
+
+            res = peer.request("getTrackers")
+            if not res or "error" in res:
+                continue
+
+            num_success += 1
+            for tracker_address in res["trackers"]:
+                if type(tracker_address) is bytes:  # Backward compatibilitys
+                    tracker_address = tracker_address.decode("utf8")
+                added = self.onTrackerFound(tracker_address)
+                if added:  # Only add one tracker from one source
+                    break
+
+        if not num_success and len(peers) < 20:
+            self.time_discover = 0.0
+
+        if num_success:
+            self.save()
+
+        self.log.debug("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s))
+
+
+if "tracker_storage" not in locals():
+    tracker_storage = TrackerStorage()
+
+
+@PluginManager.registerTo("SiteAnnouncer")
+class SiteAnnouncerPlugin(object):
+    def getTrackers(self):
+        if tracker_storage.time_discover < time.time() - 5 * 60:
+            tracker_storage.time_discover = time.time()
+            gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers())
+        trackers = super(SiteAnnouncerPlugin, self).getTrackers()
+        shared_trackers = list(tracker_storage.getTrackers("shared").keys())
+        if shared_trackers:
+            return trackers + shared_trackers
+        else:
+            return trackers
+
+    def announceTracker(self, tracker, *args, **kwargs):
+        res = super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs)
+        if res:
+            latency = res
+            tracker_storage.onTrackerSuccess(tracker, latency)
+        elif res is False:
+            tracker_storage.onTrackerError(tracker)
+
+        return res
+
+
+@PluginManager.registerTo("FileRequest")
+class FileRequestPlugin(object):
+    def actionGetTrackers(self, params):
+        shared_trackers = list(tracker_storage.getWorkingTrackers("shared").keys())
+        self.response({"trackers": shared_trackers})
+
+
+@PluginManager.registerTo("FileServer")
+class FileServerPlugin(object):
+    def portCheck(self, *args, **kwargs):
+        res = super(FileServerPlugin, self).portCheck(*args, **kwargs)
+        if res and not config.tor == "always" and "Bootstrapper" in PluginManager.plugin_manager.plugin_names:
+            for ip in self.ip_external_list:
+                my_tracker_address = "zero://%s:%s" % (ip, config.fileserver_port)
+                tracker_storage.onTrackerFound(my_tracker_address, my=True)
+        return res
+
+
+@PluginManager.registerTo("ConfigPlugin")
+class ConfigPlugin(object):
+    def createArguments(self):
+        group = self.parser.add_argument_group("AnnounceShare plugin")
+        group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached', default=5, type=int, metavar='limit')
+
+        return super(ConfigPlugin, self).createArguments()

plugins/AnnounceShare/Test/TestAnnounceShare.py

@@ -0,0 +1,24 @@
+import pytest
+
+from AnnounceShare import AnnounceSharePlugin
+from Peer import Peer
+from Config import config
+
+
+@pytest.mark.usefixtures("resetSettings")
+@pytest.mark.usefixtures("resetTempSettings")
+class TestAnnounceShare:
+    def testAnnounceList(self, file_server):
+        open("%s/trackers.json" % config.data_dir, "w").write("{}")
+        tracker_storage = AnnounceSharePlugin.tracker_storage
+        tracker_storage.load()
+        peer = Peer(file_server.ip, 1544, connection_server=file_server)
+        assert peer.request("getTrackers")["trackers"] == []
+
+        tracker_storage.onTrackerFound("zero://%s:15441" % file_server.ip)
+        assert peer.request("getTrackers")["trackers"] == []
+
+        # It needs to have at least one successfull announce to be shared to other peers
+        tracker_storage.onTrackerSuccess("zero://%s:15441" % file_server.ip, 1.0)
+        assert peer.request("getTrackers")["trackers"] == ["zero://%s:15441" % file_server.ip]
+

plugins/AnnounceShare/Test/conftest.py

@@ -0,0 +1,3 @@
+from src.Test.conftest import *
+
+from Config import config

plugins/AnnounceShare/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/AnnounceShare/__init__.py

@@ -0,0 +1 @@
+from . import AnnounceSharePlugin

plugins/AnnounceZero/AnnounceZeroPlugin.py

@@ -0,0 +1,138 @@
+import time
+import itertools
+
+from Plugin import PluginManager
+from util import helper
+from Crypt import CryptRsa
+
+allow_reload = False  # No source reload supported in this plugin
+time_full_announced = {}  # Tracker address: Last announced all site to tracker
+connection_pool = {}  # Tracker address: Peer object
+
+
+# We can only import plugin host clases after the plugins are loaded
+@PluginManager.afterLoad
+def importHostClasses():
+    global Peer, AnnounceError
+    from Peer import Peer
+    from Site.SiteAnnouncer import AnnounceError
+
+
+# Process result got back from tracker
+def processPeerRes(tracker_address, site, peers):
+    added = 0
+    # Ip4
+    found_ipv4 = 0
+    peers_normal = itertools.chain(peers.get("ip4", []), peers.get("ipv4", []), peers.get("ipv6", []))
+    for packed_address in peers_normal:
+        found_ipv4 += 1
+        peer_ip, peer_port = helper.unpackAddress(packed_address)
+        if site.addPeer(peer_ip, peer_port, source="tracker"):
+            added += 1
+    # Onion
+    found_onion = 0
+    for packed_address in peers["onion"]:
+        found_onion += 1
+        peer_onion, peer_port = helper.unpackOnionAddress(packed_address)
+        if site.addPeer(peer_onion, peer_port, source="tracker"):
+            added += 1
+
+    if added:
+        site.worker_manager.onPeers()
+        site.updateWebsocket(peers_added=added)
+    return added
+
+
+@PluginManager.registerTo("SiteAnnouncer")
+class SiteAnnouncerPlugin(object):
+    def getTrackerHandler(self, protocol):
+        if protocol == "zero":
+            return self.announceTrackerZero
+        else:
+            return super(SiteAnnouncerPlugin, self).getTrackerHandler(protocol)
+
+    def announceTrackerZero(self, tracker_address, mode="start", num_want=10):
+        global time_full_announced
+        s = time.time()
+
+        need_types = ["ip4"]   # ip4 for backward compatibility reasons
+        need_types += self.site.connection_server.supported_ip_types
+        if self.site.connection_server.tor_manager.enabled:
+            need_types.append("onion")
+
+        if mode == "start" or mode == "more":  # Single: Announce only this site
+            sites = [self.site]
+            full_announce = False
+        else:  # Multi: Announce all currently serving site
+            full_announce = True
+            if time.time() - time_full_announced.get(tracker_address, 0) < 60 * 15:  # No reannounce all sites within short time
+                return None
+            time_full_announced[tracker_address] = time.time()
+            from Site import SiteManager
+            sites = [site for site in SiteManager.site_manager.sites.values() if site.isServing()]
+
+        # Create request
+        add_types = self.getOpenedServiceTypes()
+        request = {
+            "hashes": [], "onions": [], "port": self.fileserver_port, "need_types": need_types, "need_num": 20, "add": add_types
+        }
+        for site in sites:
+            if "onion" in add_types:
+                onion = self.site.connection_server.tor_manager.getOnion(site.address)
+                request["onions"].append(onion)
+            request["hashes"].append(site.address_hash)
+
+        # Tracker can remove sites that we don't announce
+        if full_announce:
+            request["delete"] = True
+
+        # Sent request to tracker
+        tracker_peer = connection_pool.get(tracker_address)  # Re-use tracker connection if possible
+        if not tracker_peer:
+            tracker_ip, tracker_port = tracker_address.rsplit(":", 1)
+            tracker_peer = Peer(str(tracker_ip), int(tracker_port), connection_server=self.site.connection_server)
+            tracker_peer.is_tracker_connection = True
+            connection_pool[tracker_address] = tracker_peer
+
+        res = tracker_peer.request("announce", request)
+
+        if not res or "peers" not in res:
+            if full_announce:
+                time_full_announced[tracker_address] = 0
+            raise AnnounceError("Invalid response: %s" % res)
+
+        # Add peers from response to site
+        site_index = 0
+        peers_added = 0
+        for site_res in res["peers"]:
+            site = sites[site_index]
+            peers_added += processPeerRes(tracker_address, site, site_res)
+            site_index += 1
+
+        # Check if we need to sign prove the onion addresses
+        if "onion_sign_this" in res:
+            self.site.log.debug("Signing %s for %s to add %s onions" % (res["onion_sign_this"], tracker_address, len(sites)))
+            request["onion_signs"] = {}
+            request["onion_sign_this"] = res["onion_sign_this"]
+            request["need_num"] = 0
+            for site in sites:
+                onion = self.site.connection_server.tor_manager.getOnion(site.address)
+                publickey = self.site.connection_server.tor_manager.getPublickey(onion)
+                if publickey not in request["onion_signs"]:
+                    sign = CryptRsa.sign(res["onion_sign_this"].encode("utf8"), self.site.connection_server.tor_manager.getPrivatekey(onion))
+                    request["onion_signs"][publickey] = sign
+            res = tracker_peer.request("announce", request)
+            if not res or "onion_sign_this" in res:
+                if full_announce:
+                    time_full_announced[tracker_address] = 0
+                raise AnnounceError("Announce onion address to failed: %s" % res)
+
+        if full_announce:
+            tracker_peer.remove()  # Close connection, we don't need it in next 5 minute
+
+        self.site.log.debug(
+            "Tracker announce result: zero://%s (sites: %s, new peers: %s, add: %s) in %.3fs" %
+            (tracker_address, site_index, peers_added, add_types, time.time() - s)
+        )
+
+        return True

plugins/AnnounceZero/__init__.py

@@ -0,0 +1 @@
+from . import AnnounceZeroPlugin

plugins/Bigfile/BigfilePiecefield.py

@@ -0,0 +1,164 @@
+import array
+
+
+def packPiecefield(data):
+    assert isinstance(data, bytes) or isinstance(data, bytearray)
+    res = []
+    if not data:
+        return array.array("H", b"")
+
+    if data[0] == b"\x00":
+        res.append(0)
+        find = b"\x01"
+    else:
+        find = b"\x00"
+    last_pos = 0
+    pos = 0
+    while 1:
+        pos = data.find(find, pos)
+        if find == b"\x00":
+            find = b"\x01"
+        else:
+            find = b"\x00"
+        if pos == -1:
+            res.append(len(data) - last_pos)
+            break
+        res.append(pos - last_pos)
+        last_pos = pos
+    return array.array("H", res)
+
+
+def unpackPiecefield(data):
+    if not data:
+        return b""
+
+    res = []
+    char = b"\x01"
+    for times in data:
+        if times > 10000:
+            return b""
+        res.append(char * times)
+        if char == b"\x01":
+            char = b"\x00"
+        else:
+            char = b"\x01"
+    return b"".join(res)
+
+
+def spliceBit(data, idx, bit):
+    assert bit == b"\x00" or bit == b"\x01"
+    if len(data) < idx:
+        data = data.ljust(idx + 1, b"\x00")
+    return data[:idx] + bit + data[idx+ 1:]
+
+class Piecefield(object):
+    def tostring(self):
+        return "".join(["1" if b else "0" for b in self.tobytes()])
+
+
+class BigfilePiecefield(Piecefield):
+    __slots__ = ["data"]
+
+    def __init__(self):
+        self.data = b""
+
+    def frombytes(self, s):
+        assert isinstance(s, bytes) or isinstance(s, bytearray)
+        self.data = s
+
+    def tobytes(self):
+        return self.data
+
+    def pack(self):
+        return packPiecefield(self.data).tobytes()
+
+    def unpack(self, s):
+        self.data = unpackPiecefield(array.array("H", s))
+
+    def __getitem__(self, key):
+        try:
+            return self.data[key]
+        except IndexError:
+            return False
+
+    def __setitem__(self, key, value):
+        self.data = spliceBit(self.data, key, value)
+
+class BigfilePiecefieldPacked(Piecefield):
+    __slots__ = ["data"]
+
+    def __init__(self):
+        self.data = b""
+
+    def frombytes(self, data):
+        assert isinstance(data, bytes) or isinstance(data, bytearray)
+        self.data = packPiecefield(data).tobytes()
+
+    def tobytes(self):
+        return unpackPiecefield(array.array("H", self.data))
+
+    def pack(self):
+        return array.array("H", self.data).tobytes()
+
+    def unpack(self, data):
+        self.data = data
+
+    def __getitem__(self, key):
+        try:
+            return self.tobytes()[key]
+        except IndexError:
+            return False
+
+    def __setitem__(self, key, value):
+        data = spliceBit(self.tobytes(), key, value)
+        self.frombytes(data)
+
+
+if __name__ == "__main__":
+    import os
+    import psutil
+    import time
+    testdata = b"\x01" * 100 + b"\x00" * 900 + b"\x01" * 4000 + b"\x00" * 4999 + b"\x01"
+    meminfo = psutil.Process(os.getpid()).memory_info
+
+    for storage in [BigfilePiecefieldPacked, BigfilePiecefield]:
+        print("-- Testing storage: %s --" % storage)
+        m = meminfo()[0]
+        s = time.time()
+        piecefields = {}
+        for i in range(10000):
+            piecefield = storage()
+            piecefield.frombytes(testdata[:i] + b"\x00" + testdata[i + 1:])
+            piecefields[i] = piecefield
+
+        print("Create x10000: +%sKB in %.3fs (len: %s)" % ((meminfo()[0] - m) / 1024, time.time() - s, len(piecefields[0].data)))
+
+        m = meminfo()[0]
+        s = time.time()
+        for piecefield in list(piecefields.values()):
+            val = piecefield[1000]
+
+        print("Query one x10000: +%sKB in %.3fs" % ((meminfo()[0] - m) / 1024, time.time() - s))
+
+        m = meminfo()[0]
+        s = time.time()
+        for piecefield in list(piecefields.values()):
+            piecefield[1000] = b"\x01"
+
+        print("Change one x10000: +%sKB in %.3fs" % ((meminfo()[0] - m) / 1024, time.time() - s))
+
+        m = meminfo()[0]
+        s = time.time()
+        for piecefield in list(piecefields.values()):
+            packed = piecefield.pack()
+
+        print("Pack x10000: +%sKB in %.3fs (len: %s)" % ((meminfo()[0] - m) / 1024, time.time() - s, len(packed)))
+
+        m = meminfo()[0]
+        s = time.time()
+        for piecefield in list(piecefields.values()):
+            piecefield.unpack(packed)
+
+        print("Unpack x10000: +%sKB in %.3fs (len: %s)" % ((meminfo()[0] - m) / 1024, time.time() - s, len(piecefields[0].data)))
+
+        piecefields = {}

plugins/Bigfile/BigfilePlugin.py

@@ -0,0 +1,784 @@
+import time
+import os
+import subprocess
+import shutil
+import collections
+import math
+import warnings
+import base64
+import binascii
+import json
+
+import gevent
+import gevent.lock
+
+from Plugin import PluginManager
+from Debug import Debug
+from Crypt import CryptHash
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore")  # Ignore missing sha3 warning
+    import merkletools
+
+from util import helper
+from util import Msgpack
+import util
+from .BigfilePiecefield import BigfilePiecefield, BigfilePiecefieldPacked
+
+
+# We can only import plugin host clases after the plugins are loaded
+@PluginManager.afterLoad
+def importPluginnedClasses():
+    global VerifyError, config
+    from Content.ContentManager import VerifyError
+    from Config import config
+
+if "upload_nonces" not in locals():
+    upload_nonces = {}
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    def isCorsAllowed(self, path):
+        if path == "/ZeroNet-Internal/BigfileUpload":
+            return True
+        else:
+            return super(UiRequestPlugin, self).isCorsAllowed(path)
+
+    @helper.encodeResponse
+    def actionBigfileUpload(self):
+        nonce = self.get.get("upload_nonce")
+        if nonce not in upload_nonces:
+            return self.error403("Upload nonce error.")
+
+        upload_info = upload_nonces[nonce]
+        del upload_nonces[nonce]
+
+        self.sendHeader(200, "text/html", noscript=True, extra_headers={
+            "Access-Control-Allow-Origin": "null",
+            "Access-Control-Allow-Credentials": "true"
+        })
+
+        self.readMultipartHeaders(self.env['wsgi.input'])  # Skip http headers
+
+        site = upload_info["site"]
+        inner_path = upload_info["inner_path"]
+
+        with site.storage.open(inner_path, "wb", create_dirs=True) as out_file:
+            merkle_root, piece_size, piecemap_info = site.content_manager.hashBigfile(
+                self.env['wsgi.input'], upload_info["size"], upload_info["piece_size"], out_file
+            )
+
+        if len(piecemap_info["sha512_pieces"]) == 1:  # Small file, don't split
+            hash = binascii.hexlify(piecemap_info["sha512_pieces"][0])
+            hash_id = site.content_manager.hashfield.getHashId(hash)
+            site.content_manager.optionalDownloaded(inner_path, hash_id, upload_info["size"], own=True)
+
+        else:  # Big file
+            file_name = helper.getFilename(inner_path)
+            site.storage.open(upload_info["piecemap"], "wb").write(Msgpack.pack({file_name: piecemap_info}))
+
+            # Find piecemap and file relative path to content.json
+            file_info = site.content_manager.getFileInfo(inner_path, new_file=True)
+            content_inner_path_dir = helper.getDirname(file_info["content_inner_path"])
+            piecemap_relative_path = upload_info["piecemap"][len(content_inner_path_dir):]
+            file_relative_path = inner_path[len(content_inner_path_dir):]
+
+            # Add file to content.json
+            if site.storage.isFile(file_info["content_inner_path"]):
+                content = site.storage.loadJson(file_info["content_inner_path"])
+            else:
+                content = {}
+            if "files_optional" not in content:
+                content["files_optional"] = {}
+
+            content["files_optional"][file_relative_path] = {
+                "sha512": merkle_root,
+                "size": upload_info["size"],
+                "piecemap": piecemap_relative_path,
+                "piece_size": piece_size
+            }
+
+            merkle_root_hash_id = site.content_manager.hashfield.getHashId(merkle_root)
+            site.content_manager.optionalDownloaded(inner_path, merkle_root_hash_id, upload_info["size"], own=True)
+            site.storage.writeJson(file_info["content_inner_path"], content)
+
+            site.content_manager.contents.loadItem(file_info["content_inner_path"])  # reload cache
+
+        return json.dumps({
+            "merkle_root": merkle_root,
+            "piece_num": len(piecemap_info["sha512_pieces"]),
+            "piece_size": piece_size,
+            "inner_path": inner_path
+        })
+
+    def readMultipartHeaders(self, wsgi_input):
+        found = False
+        for i in range(100):
+            line = wsgi_input.readline()
+            if line == b"\r\n":
+                found = True
+                break
+        if not found:
+            raise Exception("No multipart header found")
+        return i
+
+    def actionFile(self, file_path, *args, **kwargs):
+        if kwargs.get("file_size", 0) > 1024 * 1024 and kwargs.get("path_parts"):  # Only check files larger than 1MB
+            path_parts = kwargs["path_parts"]
+            site = self.server.site_manager.get(path_parts["address"])
+            big_file = site.storage.openBigfile(path_parts["inner_path"], prebuffer=2 * 1024 * 1024)
+            if big_file:
+                kwargs["file_obj"] = big_file
+                kwargs["file_size"] = big_file.size
+
+        return super(UiRequestPlugin, self).actionFile(file_path, *args, **kwargs)
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def actionBigfileUploadInit(self, to, inner_path, size):
+        valid_signers = self.site.content_manager.getValidSigners(inner_path)
+        auth_address = self.user.getAuthAddress(self.site.address)
+        if not self.site.settings["own"] and auth_address not in valid_signers:
+            self.log.error("FileWrite forbidden %s not in valid_signers %s" % (auth_address, valid_signers))
+            return self.response(to, {"error": "Forbidden, you can only modify your own files"})
+
+        nonce = CryptHash.random()
+        piece_size = 1024 * 1024
+        inner_path = self.site.content_manager.sanitizePath(inner_path)
+        file_info = self.site.content_manager.getFileInfo(inner_path, new_file=True)
+
+        content_inner_path_dir = helper.getDirname(file_info["content_inner_path"])
+        file_relative_path = inner_path[len(content_inner_path_dir):]
+
+        upload_nonces[nonce] = {
+            "added": time.time(),
+            "site": self.site,
+            "inner_path": inner_path,
+            "websocket_client": self,
+            "size": size,
+            "piece_size": piece_size,
+            "piecemap": inner_path + ".piecemap.msgpack"
+        }
+        return {
+            "url": "/ZeroNet-Internal/BigfileUpload?upload_nonce=" + nonce,
+            "piece_size": piece_size,
+            "inner_path": inner_path,
+            "file_relative_path": file_relative_path
+        }
+
+    def actionSiteSetAutodownloadBigfileLimit(self, to, limit):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        self.site.settings["autodownload_bigfile_size_limit"] = int(limit)
+        self.response(to, "ok")
+
+    def actionFileDelete(self, to, inner_path):
+        piecemap_inner_path = inner_path + ".piecemap.msgpack"
+        if self.hasFilePermission(inner_path) and self.site.storage.isFile(piecemap_inner_path):
+            # Also delete .piecemap.msgpack file if exists
+            self.log.debug("Deleting piecemap: %s" % piecemap_inner_path)
+            file_info = self.site.content_manager.getFileInfo(piecemap_inner_path)
+            if file_info:
+                content_json = self.site.storage.loadJson(file_info["content_inner_path"])
+                relative_path = file_info["relative_path"]
+                if relative_path in content_json.get("files_optional", {}):
+                    del content_json["files_optional"][relative_path]
+                    self.site.storage.writeJson(file_info["content_inner_path"], content_json)
+                    self.site.content_manager.loadContent(file_info["content_inner_path"], add_bad_files=False, force=True)
+                    try:
+                        self.site.storage.delete(piecemap_inner_path)
+                    except Exception as err:
+                        self.log.error("File %s delete error: %s" % (piecemap_inner_path, err))
+
+        return super(UiWebsocketPlugin, self).actionFileDelete(to, inner_path)
+
+
+@PluginManager.registerTo("ContentManager")
+class ContentManagerPlugin(object):
+    def getFileInfo(self, inner_path, *args, **kwargs):
+        if "|" not in inner_path:
+            return super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs)
+
+        inner_path, file_range = inner_path.split("|")
+        pos_from, pos_to = map(int, file_range.split("-"))
+        file_info = super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs)
+        return file_info
+
+    def readFile(self, file_in, size, buff_size=1024 * 64):
+        part_num = 0
+        recv_left = size
+
+        while 1:
+            part_num += 1
+            read_size = min(buff_size, recv_left)
+            part = file_in.read(read_size)
+
+            if not part:
+                break
+            yield part
+
+            if part_num % 100 == 0:  # Avoid blocking ZeroNet execution during upload
+                time.sleep(0.001)
+
+            recv_left -= read_size
+            if recv_left <= 0:
+                break
+
+    def hashBigfile(self, file_in, size, piece_size=1024 * 1024, file_out=None):
+        self.site.settings["has_bigfile"] = True
+
+        recv = 0
+        try:
+            piece_hash = CryptHash.sha512t()
+            piece_hashes = []
+            piece_recv = 0
+
+            mt = merkletools.MerkleTools()
+            mt.hash_function = CryptHash.sha512t
+
+            part = ""
+            for part in self.readFile(file_in, size):
+                if file_out:
+                    file_out.write(part)
+
+                recv += len(part)
+                piece_recv += len(part)
+                piece_hash.update(part)
+                if piece_recv >= piece_size:
+                    piece_digest = piece_hash.digest()
+                    piece_hashes.append(piece_digest)
+                    mt.leaves.append(piece_digest)
+                    piece_hash = CryptHash.sha512t()
+                    piece_recv = 0
+
+                    if len(piece_hashes) % 100 == 0 or recv == size:
+                        self.log.info("- [HASHING:%.0f%%] Pieces: %s, %.1fMB/%.1fMB" % (
+                            float(recv) / size * 100, len(piece_hashes), recv / 1024 / 1024, size / 1024 / 1024
+                        ))
+                        part = ""
+            if len(part) > 0:
+                piece_digest = piece_hash.digest()
+                piece_hashes.append(piece_digest)
+                mt.leaves.append(piece_digest)
+        except Exception as err:
+            raise err
+        finally:
+            if file_out:
+                file_out.close()
+
+        mt.make_tree()
+        merkle_root = mt.get_merkle_root()
+        if type(merkle_root) is bytes:  # Python <3.5
+            merkle_root = merkle_root.decode()
+        return merkle_root, piece_size, {
+            "sha512_pieces": piece_hashes
+        }
+
+    def hashFile(self, dir_inner_path, file_relative_path, optional=False):
+        inner_path = dir_inner_path + file_relative_path
+
+        file_size = self.site.storage.getSize(inner_path)
+        # Only care about optional files >1MB
+        if not optional or file_size < 1 * 1024 * 1024:
+            return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional)
+
+        back = {}
+        content = self.contents.get(dir_inner_path + "content.json")
+
+        hash = None
+        piecemap_relative_path = None
+        piece_size = None
+
+        # Don't re-hash if it's already in content.json
+        if content and file_relative_path in content.get("files_optional", {}):
+            file_node = content["files_optional"][file_relative_path]
+            if file_node["size"] == file_size:
+                self.log.info("- [SAME SIZE] %s" % file_relative_path)
+                hash = file_node.get("sha512")
+                piecemap_relative_path = file_node.get("piecemap")
+                piece_size = file_node.get("piece_size")
+
+        if not hash or not piecemap_relative_path:  # Not in content.json yet
+            if file_size < 5 * 1024 * 1024:  # Don't create piecemap automatically for files smaller than 5MB
+                return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional)
+
+            self.log.info("- [HASHING] %s" % file_relative_path)
+            merkle_root, piece_size, piecemap_info = self.hashBigfile(self.site.storage.open(inner_path, "rb"), file_size)
+            if not hash:
+                hash = merkle_root
+
+            if not piecemap_relative_path:
+                file_name = helper.getFilename(file_relative_path)
+                piecemap_relative_path = file_relative_path + ".piecemap.msgpack"
+                piecemap_inner_path = inner_path + ".piecemap.msgpack"
+
+                self.site.storage.open(piecemap_inner_path, "wb").write(Msgpack.pack({file_name: piecemap_info}))
+
+                back.update(super(ContentManagerPlugin, self).hashFile(dir_inner_path, piecemap_relative_path, optional=True))
+
+        piece_num = int(math.ceil(float(file_size) / piece_size))
+
+        # Add the merkle root to hashfield
+        hash_id = self.site.content_manager.hashfield.getHashId(hash)
+        self.optionalDownloaded(inner_path, hash_id, file_size, own=True)
+        self.site.storage.piecefields[hash].frombytes(b"\x01" * piece_num)
+
+        back[file_relative_path] = {"sha512": hash, "size": file_size, "piecemap": piecemap_relative_path, "piece_size": piece_size}
+        return back
+
+    def getPiecemap(self, inner_path):
+        file_info = self.site.content_manager.getFileInfo(inner_path)
+        piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"]
+        self.site.needFile(piecemap_inner_path, priority=20)
+        piecemap = Msgpack.unpack(self.site.storage.open(piecemap_inner_path, "rb").read())[helper.getFilename(inner_path)]
+        piecemap["piece_size"] = file_info["piece_size"]
+        return piecemap
+
+    def verifyPiece(self, inner_path, pos, piece):
+        piecemap = self.getPiecemap(inner_path)
+        piece_i = int(pos / piecemap["piece_size"])
+        if CryptHash.sha512sum(piece, format="digest") != piecemap["sha512_pieces"][piece_i]:
+            raise VerifyError("Invalid hash")
+        return True
+
+    def verifyFile(self, inner_path, file, ignore_same=True):
+        if "|" not in inner_path:
+            return super(ContentManagerPlugin, self).verifyFile(inner_path, file, ignore_same)
+
+        inner_path, file_range = inner_path.split("|")
+        pos_from, pos_to = map(int, file_range.split("-"))
+
+        return self.verifyPiece(inner_path, pos_from, file)
+
+    def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):
+        if "|" in inner_path:
+            inner_path, file_range = inner_path.split("|")
+            pos_from, pos_to = map(int, file_range.split("-"))
+            file_info = self.getFileInfo(inner_path)
+
+            # Mark piece downloaded
+            piece_i = int(pos_from / file_info["piece_size"])
+            self.site.storage.piecefields[file_info["sha512"]][piece_i] = b"\x01"
+
+            # Only add to site size on first request
+            if hash_id in self.hashfield:
+                size = 0
+        elif size > 1024 * 1024:
+            file_info = self.getFileInfo(inner_path)
+            if file_info and "sha512" in file_info:  # We already have the file, but not in piecefield
+                sha512 = file_info["sha512"]
+                if sha512 not in self.site.storage.piecefields:
+                    self.site.storage.checkBigfile(inner_path)
+
+        return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own)
+
+    def optionalRemoved(self, inner_path, hash_id, size=None):
+        if size and size > 1024 * 1024:
+            file_info = self.getFileInfo(inner_path)
+            sha512 = file_info["sha512"]
+            if sha512 in self.site.storage.piecefields:
+                del self.site.storage.piecefields[sha512]
+
+            # Also remove other pieces of the file from download queue
+            for key in list(self.site.bad_files.keys()):
+                if key.startswith(inner_path + "|"):
+                    del self.site.bad_files[key]
+            self.site.worker_manager.removeSolvedFileTasks()
+        return super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size)
+
+
+@PluginManager.registerTo("SiteStorage")
+class SiteStoragePlugin(object):
+    def __init__(self, *args, **kwargs):
+        super(SiteStoragePlugin, self).__init__(*args, **kwargs)
+        self.piecefields = collections.defaultdict(BigfilePiecefield)
+        if "piecefields" in self.site.settings.get("cache", {}):
+            for sha512, piecefield_packed in self.site.settings["cache"].get("piecefields").items():
+                if piecefield_packed:
+                    self.piecefields[sha512].unpack(base64.b64decode(piecefield_packed))
+            self.site.settings["cache"]["piecefields"] = {}
+
+    def createSparseFile(self, inner_path, size, sha512=None):
+        file_path = self.getPath(inner_path)
+
+        file_dir = os.path.dirname(file_path)
+        if not os.path.isdir(file_dir):
+            os.makedirs(file_dir)
+
+        f = open(file_path, 'wb')
+        f.truncate(min(1024 * 1024 * 5, size))  # Only pre-allocate up to 5MB
+        f.close()
+        if os.name == "nt":
+            startupinfo = subprocess.STARTUPINFO()
+            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            subprocess.call(["fsutil", "sparse", "setflag", file_path], close_fds=True, startupinfo=startupinfo)
+
+        if sha512 and sha512 in self.piecefields:
+            self.log.debug("%s: File not exists, but has piecefield. Deleting piecefield." % inner_path)
+            del self.piecefields[sha512]
+
+    def write(self, inner_path, content):
+        if "|" not in inner_path:
+            return super(SiteStoragePlugin, self).write(inner_path, content)
+
+        # Write to specific position by passing |{pos} after the filename
+        inner_path, file_range = inner_path.split("|")
+        pos_from, pos_to = map(int, file_range.split("-"))
+        file_path = self.getPath(inner_path)
+
+        # Create dir if not exist
+        file_dir = os.path.dirname(file_path)
+        if not os.path.isdir(file_dir):
+            os.makedirs(file_dir)
+
+        if not os.path.isfile(file_path):
+            file_info = self.site.content_manager.getFileInfo(inner_path)
+            self.createSparseFile(inner_path, file_info["size"])
+
+        # Write file
+        with open(file_path, "rb+") as file:
+            file.seek(pos_from)
+            if hasattr(content, 'read'):  # File-like object
+                shutil.copyfileobj(content, file)  # Write buff to disk
+            else:  # Simple string
+                file.write(content)
+        del content
+        self.onUpdated(inner_path)
+
+    def checkBigfile(self, inner_path):
+        file_info = self.site.content_manager.getFileInfo(inner_path)
+        if not file_info or (file_info and "piecemap" not in file_info):  # It's not a big file
+            return False
+
+        self.site.settings["has_bigfile"] = True
+        file_path = self.getPath(inner_path)
+        sha512 = file_info["sha512"]
+        piece_num = int(math.ceil(float(file_info["size"]) / file_info["piece_size"]))
+        if os.path.isfile(file_path):
+            if sha512 not in self.piecefields:
+                if open(file_path, "rb").read(128) == b"\0" * 128:
+                    piece_data = b"\x00"
+                else:
+                    piece_data = b"\x01"
+                self.log.debug("%s: File exists, but not in piecefield. Filling piecefiled with %s * %s." % (inner_path, piece_num, piece_data))
+                self.piecefields[sha512].frombytes(piece_data * piece_num)
+        else:
+            self.log.debug("Creating bigfile: %s" % inner_path)
+            self.createSparseFile(inner_path, file_info["size"], sha512)
+            self.piecefields[sha512].frombytes(b"\x00" * piece_num)
+            self.log.debug("Created bigfile: %s" % inner_path)
+        return True
+
+    def openBigfile(self, inner_path, prebuffer=0):
+        if not self.checkBigfile(inner_path):
+            return False
+        self.site.needFile(inner_path, blocking=False)  # Download piecemap
+        return BigFile(self.site, inner_path, prebuffer=prebuffer)
+
+
+class BigFile(object):
+    def __init__(self, site, inner_path, prebuffer=0):
+        self.site = site
+        self.inner_path = inner_path
+        file_path = site.storage.getPath(inner_path)
+        file_info = self.site.content_manager.getFileInfo(inner_path)
+        self.piece_size = file_info["piece_size"]
+        self.sha512 = file_info["sha512"]
+        self.size = file_info["size"]
+        self.prebuffer = prebuffer
+        self.read_bytes = 0
+
+        self.piecefield = self.site.storage.piecefields[self.sha512]
+        self.f = open(file_path, "rb+")
+        self.read_lock = gevent.lock.Semaphore()
+
+    def read(self, buff=64 * 1024):
+        with self.read_lock:
+            pos = self.f.tell()
+            read_until = min(self.size, pos + buff)
+            requests = []
+            # Request all required blocks
+            while 1:
+                piece_i = int(pos / self.piece_size)
+                if piece_i * self.piece_size >= read_until:
+                    break
+                pos_from = piece_i * self.piece_size
+                pos_to = pos_from + self.piece_size
+                if not self.piecefield[piece_i]:
+                    requests.append(self.site.needFile("%s|%s-%s" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=10))
+                pos += self.piece_size
+
+            if not all(requests):
+                return None
+
+            # Request prebuffer
+            if self.prebuffer:
+                prebuffer_until = min(self.size, read_until + self.prebuffer)
+                priority = 3
+                while 1:
+                    piece_i = int(pos / self.piece_size)
+                    if piece_i * self.piece_size >= prebuffer_until:
+                        break
+                    pos_from = piece_i * self.piece_size
+                    pos_to = pos_from + self.piece_size
+                    if not self.piecefield[piece_i]:
+                        self.site.needFile("%s|%s-%s" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=max(0, priority))
+                    priority -= 1
+                    pos += self.piece_size
+
+            gevent.joinall(requests)
+            self.read_bytes += buff
+
+            # Increase buffer for long reads
+            if self.read_bytes > 7 * 1024 * 1024 and self.prebuffer < 5 * 1024 * 1024:
+                self.site.log.debug("%s: Increasing bigfile buffer size to 5MB..." % self.inner_path)
+                self.prebuffer = 5 * 1024 * 1024
+
+            return self.f.read(buff)
+
+    def seek(self, pos, whence=0):
+        with self.read_lock:
+            if whence == 2:  # Relative from file end
+                pos = self.size + pos  # Use the real size instead of size on the disk
+                whence = 0
+            return self.f.seek(pos, whence)
+
+    def tell(self):
+        return self.f.tell()
+
+    def close(self):
+        self.f.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
+@PluginManager.registerTo("WorkerManager")
+class WorkerManagerPlugin(object):
+    def addTask(self, inner_path, *args, **kwargs):
+        file_info = kwargs.get("file_info")
+        if file_info and "piecemap" in file_info:  # Bigfile
+            self.site.settings["has_bigfile"] = True
+
+            piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"]
+            piecemap_task = None
+            if not self.site.storage.isFile(piecemap_inner_path):
+                # Start download piecemap
+                piecemap_task = super(WorkerManagerPlugin, self).addTask(piecemap_inner_path, priority=30)
+                autodownload_bigfile_size_limit = self.site.settings.get("autodownload_bigfile_size_limit", config.autodownload_bigfile_size_limit)
+                if "|" not in inner_path and self.site.isDownloadable(inner_path) and file_info["size"] / 1024 / 1024 <= autodownload_bigfile_size_limit:
+                    gevent.spawn_later(0.1, self.site.needFile, inner_path + "|all")  # Download all pieces
+
+            if "|" in inner_path:
+                # Start download piece
+                task = super(WorkerManagerPlugin, self).addTask(inner_path, *args, **kwargs)
+
+                inner_path, file_range = inner_path.split("|")
+                pos_from, pos_to = map(int, file_range.split("-"))
+                task["piece_i"] = int(pos_from / file_info["piece_size"])
+                task["sha512"] = file_info["sha512"]
+            else:
+                if inner_path in self.site.bad_files:
+                    del self.site.bad_files[inner_path]
+                if piecemap_task:
+                    task = piecemap_task
+                else:
+                    fake_evt = gevent.event.AsyncResult()  # Don't download anything if no range specified
+                    fake_evt.set(True)
+                    task = {"evt": fake_evt}
+
+            if not self.site.storage.isFile(inner_path):
+                self.site.storage.createSparseFile(inner_path, file_info["size"], file_info["sha512"])
+                piece_num = int(math.ceil(float(file_info["size"]) / file_info["piece_size"]))
+                self.site.storage.piecefields[file_info["sha512"]].frombytes(b"\x00" * piece_num)
+        else:
+            task = super(WorkerManagerPlugin, self).addTask(inner_path, *args, **kwargs)
+        return task
+
+    def taskAddPeer(self, task, peer):
+        if "piece_i" in task:
+            if not peer.piecefields[task["sha512"]][task["piece_i"]]:
+                if task["sha512"] not in peer.piecefields:
+                    gevent.spawn(peer.updatePiecefields, force=True)
+                elif not task["peers"]:
+                    gevent.spawn(peer.updatePiecefields)
+
+                return False  # Deny to add peers to task if file not in piecefield
+        return super(WorkerManagerPlugin, self).taskAddPeer(task, peer)
+
+
+@PluginManager.registerTo("FileRequest")
+class FileRequestPlugin(object):
+    def isReadable(self, site, inner_path, file, pos):
+        # Peek into file
+        if file.read(10) == b"\0" * 10:
+            # Looks empty, but makes sures we don't have that piece
+            file_info = site.content_manager.getFileInfo(inner_path)
+            if "piece_size" in file_info:
+                piece_i = int(pos / file_info["piece_size"])
+                if not site.storage.piecefields[file_info["sha512"]][piece_i]:
+                    return False
+        # Seek back to position we want to read
+        file.seek(pos)
+        return super(FileRequestPlugin, self).isReadable(site, inner_path, file, pos)
+
+    def actionGetPiecefields(self, params):
+        site = self.sites.get(params["site"])
+        if not site or not site.isServing():  # Site unknown or not serving
+            self.response({"error": "Unknown site"})
+            return False
+
+        # Add peer to site if not added before
+        peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True)
+        if not peer.connection:  # Just added
+            peer.connect(self.connection)  # Assign current connection to peer
+
+        piecefields_packed = {sha512: piecefield.pack() for sha512, piecefield in site.storage.piecefields.items()}
+        self.response({"piecefields_packed": piecefields_packed})
+
+    def actionSetPiecefields(self, params):
+        site = self.sites.get(params["site"])
+        if not site or not site.isServing():  # Site unknown or not serving
+            self.response({"error": "Unknown site"})
+            self.connection.badAction(5)
+            return False
+
+        # Add or get peer
+        peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, connection=self.connection)
+        if not peer.connection:
+            peer.connect(self.connection)
+
+        peer.piecefields = collections.defaultdict(BigfilePiecefieldPacked)
+        for sha512, piecefield_packed in params["piecefields_packed"].items():
+            peer.piecefields[sha512].unpack(piecefield_packed)
+        site.settings["has_bigfile"] = True
+
+        self.response({"ok": "Updated"})
+
+
+@PluginManager.registerTo("Peer")
+class PeerPlugin(object):
+    def __getattr__(self, key):
+        if key == "piecefields":
+            self.piecefields = collections.defaultdict(BigfilePiecefieldPacked)
+            return self.piecefields
+        elif key == "time_piecefields_updated":
+            self.time_piecefields_updated = None
+            return self.time_piecefields_updated
+        else:
+            return super(PeerPlugin, self).__getattr__(key)
+
+    @util.Noparallel(ignore_args=True)
+    def updatePiecefields(self, force=False):
+        if self.connection and self.connection.handshake.get("rev", 0) < 2190:
+            return False  # Not supported
+
+        # Don't update piecefield again in 1 min
+        if self.time_piecefields_updated and time.time() - self.time_piecefields_updated < 60 and not force:
+            return False
+
+        self.time_piecefields_updated = time.time()
+        res = self.request("getPiecefields", {"site": self.site.address})
+        if not res or "error" in res:
+            return False
+
+        self.piecefields = collections.defaultdict(BigfilePiecefieldPacked)
+        try:
+            for sha512, piecefield_packed in res["piecefields_packed"].items():
+                self.piecefields[sha512].unpack(piecefield_packed)
+        except Exception as err:
+            self.log("Invalid updatePiecefields response: %s" % Debug.formatException(err))
+
+        return self.piecefields
+
+    def sendMyHashfield(self, *args, **kwargs):
+        return super(PeerPlugin, self).sendMyHashfield(*args, **kwargs)
+
+    def updateHashfield(self, *args, **kwargs):
+        if self.site.settings.get("has_bigfile"):
+            thread = gevent.spawn(self.updatePiecefields, *args, **kwargs)
+            back = super(PeerPlugin, self).updateHashfield(*args, **kwargs)
+            thread.join()
+            return back
+        else:
+            return super(PeerPlugin, self).updateHashfield(*args, **kwargs)
+
+    def getFile(self, site, inner_path, *args, **kwargs):
+        if "|" in inner_path:
+            inner_path, file_range = inner_path.split("|")
+            pos_from, pos_to = map(int, file_range.split("-"))
+            kwargs["pos_from"] = pos_from
+            kwargs["pos_to"] = pos_to
+        return super(PeerPlugin, self).getFile(site, inner_path, *args, **kwargs)
+
+
+@PluginManager.registerTo("Site")
+class SitePlugin(object):
+    def isFileDownloadAllowed(self, inner_path, file_info):
+        if "piecemap" in file_info:
+            file_size_mb = file_info["size"] / 1024 / 1024
+            if config.bigfile_size_limit and file_size_mb > config.bigfile_size_limit:
+                self.log.debug(
+                    "Bigfile size %s too large: %sMB > %sMB, skipping..." %
+                    (inner_path, file_size_mb, config.bigfile_size_limit)
+                )
+                return False
+
+            file_info = file_info.copy()
+            file_info["size"] = file_info["piece_size"]
+        return super(SitePlugin, self).isFileDownloadAllowed(inner_path, file_info)
+
+    def getSettingsCache(self):
+        back = super(SitePlugin, self).getSettingsCache()
+        if self.storage.piecefields:
+            back["piecefields"] = {sha512: base64.b64encode(piecefield.pack()).decode("utf8") for sha512, piecefield in self.storage.piecefields.items()}
+        return back
+
+    def needFile(self, inner_path, *args, **kwargs):
+        if inner_path.endswith("|all"):
+            @util.Pooled(20)
+            def pooledNeedBigfile(inner_path, *args, **kwargs):
+                if inner_path not in self.bad_files:
+                    self.log.debug("Cancelled piece, skipping %s" % inner_path)
+                    return False
+                return self.needFile(inner_path, *args, **kwargs)
+
+            inner_path = inner_path.replace("|all", "")
+            file_info = self.needFileInfo(inner_path)
+            file_size = file_info["size"]
+            piece_size = file_info["piece_size"]
+
+            piece_num = int(math.ceil(float(file_size) / piece_size))
+
+            file_threads = []
+
+            piecefield = self.storage.piecefields.get(file_info["sha512"])
+
+            for piece_i in range(piece_num):
+                piece_from = piece_i * piece_size
+                piece_to = min(file_size, piece_from + piece_size)
+                if not piecefield or not piecefield[piece_i]:
+                    inner_path_piece = "%s|%s-%s" % (inner_path, piece_from, piece_to)
+                    self.bad_files[inner_path_piece] = self.bad_files.get(inner_path_piece, 1)
+                    res = pooledNeedBigfile(inner_path_piece, blocking=False)
+                    if res is not True and res is not False:
+                        file_threads.append(res)
+            gevent.joinall(file_threads)
+        else:
+            return super(SitePlugin, self).needFile(inner_path, *args, **kwargs)
+
+
+@PluginManager.registerTo("ConfigPlugin")
+class ConfigPlugin(object):
+    def createArguments(self):
+        group = self.parser.add_argument_group("Bigfile plugin")
+        group.add_argument('--autodownload_bigfile_size_limit', help='Also download bigfiles smaller than this limit if help distribute option is checked', default=1, metavar="MB", type=int)
+        group.add_argument('--bigfile_size_limit', help='Maximum size of downloaded big files', default=False, metavar="MB", type=int)
+
+        return super(ConfigPlugin, self).createArguments()

plugins/Bigfile/Test/TestBigfile.py

@@ -0,0 +1,574 @@
+import time
+import io
+import binascii
+
+import pytest
+import mock
+
+from Connection import ConnectionServer
+from Content.ContentManager import VerifyError
+from File import FileServer
+from File import FileRequest
+from Worker import WorkerManager
+from Peer import Peer
+from Bigfile import BigfilePiecefield, BigfilePiecefieldPacked
+from Test import Spy
+from util import Msgpack
+
+
+@pytest.mark.usefixtures("resetSettings")
+@pytest.mark.usefixtures("resetTempSettings")
+class TestBigfile:
+    privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"
+    piece_size = 1024 * 1024
+
+    def createBigfile(self, site, inner_path="data/optional.any.iso", pieces=10):
+        f = site.storage.open(inner_path, "w")
+        for i in range(pieces * 100):
+            f.write(("Test%s" % i).ljust(10, "-") * 1000)
+        f.close()
+        assert site.content_manager.sign("content.json", self.privatekey)
+        return inner_path
+
+    def testPiecemapCreate(self, site):
+        inner_path = self.createBigfile(site)
+        content = site.storage.loadJson("content.json")
+        assert "data/optional.any.iso" in content["files_optional"]
+        file_node = content["files_optional"][inner_path]
+        assert file_node["size"] == 10 * 1000 * 1000
+        assert file_node["sha512"] == "47a72cde3be80b4a829e7674f72b7c6878cf6a70b0c58c6aa6c17d7e9948daf6"
+        assert file_node["piecemap"] == inner_path + ".piecemap.msgpack"
+
+        piecemap = Msgpack.unpack(site.storage.open(file_node["piecemap"], "rb").read())["optional.any.iso"]
+        assert len(piecemap["sha512_pieces"]) == 10
+        assert piecemap["sha512_pieces"][0] != piecemap["sha512_pieces"][1]
+        assert binascii.hexlify(piecemap["sha512_pieces"][0]) == b"a73abad9992b3d0b672d0c2a292046695d31bebdcb1e150c8410bbe7c972eff3"
+
+    def testVerifyPiece(self, site):
+        inner_path = self.createBigfile(site)
+
+        # Verify all 10 piece
+        f = site.storage.open(inner_path, "rb")
+        for i in range(10):
+            piece = io.BytesIO(f.read(1024 * 1024))
+            piece.seek(0)
+            site.content_manager.verifyPiece(inner_path, i * 1024 * 1024, piece)
+        f.close()
+
+        # Try to verify piece 0 with piece 1 hash
+        with pytest.raises(VerifyError) as err:
+            i = 1
+            f = site.storage.open(inner_path, "rb")
+            piece = io.BytesIO(f.read(1024 * 1024))
+            f.close()
+            site.content_manager.verifyPiece(inner_path, i * 1024 * 1024, piece)
+        assert "Invalid hash" in str(err)
+
+    def testSparseFile(self, site):
+        inner_path = "sparsefile"
+
+        # Create a 100MB sparse file
+        site.storage.createSparseFile(inner_path, 100 * 1024 * 1024)
+
+        # Write to file beginning
+        s = time.time()
+        f = site.storage.write("%s|%s-%s" % (inner_path, 0, 1024 * 1024), b"hellostart" * 1024)
+        time_write_start = time.time() - s
+
+        # Write to file end
+        s = time.time()
+        f = site.storage.write("%s|%s-%s" % (inner_path, 99 * 1024 * 1024, 99 * 1024 * 1024 + 1024 * 1024), b"helloend" * 1024)
+        time_write_end = time.time() - s
+
+        # Verify writes
+        f = site.storage.open(inner_path)
+        assert f.read(10) == b"hellostart"
+        f.seek(99 * 1024 * 1024)
+        assert f.read(8) == b"helloend"
+        f.close()
+
+        site.storage.delete(inner_path)
+
+        # Writing to end shold not take much longer, than writing to start
+        assert time_write_end <= max(0.1, time_write_start * 1.1)
+
+    def testRangedFileRequest(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        file_server.sites[site.address] = site
+        client = FileServer(file_server.ip, 1545)
+        client.sites[site_temp.address] = site_temp
+        site_temp.connection_server = client
+        connection = client.getConnection(file_server.ip, 1544)
+
+        # Add file_server as peer to client
+        peer_file_server = site_temp.addPeer(file_server.ip, 1544)
+
+        buff = peer_file_server.getFile(site_temp.address, "%s|%s-%s" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024))
+
+        assert len(buff.getvalue()) == 1 * 1024 * 1024  # Correct block size
+        assert buff.getvalue().startswith(b"Test524")  # Correct data
+        buff.seek(0)
+        assert site.content_manager.verifyPiece(inner_path, 5 * 1024 * 1024, buff)  # Correct hash
+
+        connection.close()
+        client.stop()
+
+    def testRangedFileDownload(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Make sure the file and the piecemap in the optional hashfield
+        file_info = site.content_manager.getFileInfo(inner_path)
+        assert site.content_manager.hashfield.hasHash(file_info["sha512"])
+
+        piecemap_hash = site.content_manager.getFileInfo(file_info["piecemap"])["sha512"]
+        assert site.content_manager.hashfield.hasHash(piecemap_hash)
+
+        # Init client server
+        client = ConnectionServer(file_server.ip, 1545)
+        site_temp.connection_server = client
+        peer_client = site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"]
+        assert not bad_files
+
+        # client_piecefield = peer_client.piecefields[file_info["sha512"]].tostring()
+        # assert client_piecefield == "1" * 10
+
+        # Download 5. and 10. block
+
+        site_temp.needFile("%s|%s-%s" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024))
+        site_temp.needFile("%s|%s-%s" % (inner_path, 9 * 1024 * 1024, 10 * 1024 * 1024))
+
+        # Verify 0. block not downloaded
+        f = site_temp.storage.open(inner_path)
+        assert f.read(10) == b"\0" * 10
+        # Verify 5. and 10. block downloaded
+        f.seek(5 * 1024 * 1024)
+        assert f.read(7) == b"Test524"
+        f.seek(9 * 1024 * 1024)
+        assert f.read(7) == b"943---T"
+
+        # Verify hashfield
+        assert set(site_temp.content_manager.hashfield) == set([18343, 43727])  # 18343: data/optional.any.iso, 43727: data/optional.any.iso.hashmap.msgpack
+
+    def testOpenBigfile(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        client = ConnectionServer(file_server.ip, 1545)
+        site_temp.connection_server = client
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        # Open virtual file
+        assert not site_temp.storage.isFile(inner_path)
+
+        with site_temp.storage.openBigfile(inner_path) as f:
+            with Spy.Spy(FileRequest, "route") as requests:
+                f.seek(5 * 1024 * 1024)
+                assert f.read(7) == b"Test524"
+
+                f.seek(9 * 1024 * 1024)
+                assert f.read(7) == b"943---T"
+
+            assert len(requests) == 4  # 1x peicemap + 1x getpiecefield + 2x for pieces
+
+            assert set(site_temp.content_manager.hashfield) == set([18343, 43727])
+
+            assert site_temp.storage.piecefields[f.sha512].tostring() == "0000010001"
+            assert f.sha512 in site_temp.getSettingsCache()["piecefields"]
+
+            # Test requesting already downloaded
+            with Spy.Spy(FileRequest, "route") as requests:
+                f.seek(5 * 1024 * 1024)
+                assert f.read(7) == b"Test524"
+
+            assert len(requests) == 0
+
+            # Test requesting multi-block overflow reads
+            with Spy.Spy(FileRequest, "route") as requests:
+                f.seek(5 * 1024 * 1024)  # We already have this block
+                data = f.read(1024 * 1024 * 3)  # Our read overflow to 6. and 7. block
+                assert data.startswith(b"Test524")
+                assert data.endswith(b"Test838-")
+                assert b"\0" not in data  # No null bytes allowed
+
+            assert len(requests) == 2  # Two block download
+
+            # Test out of range request
+            f.seek(5 * 1024 * 1024)
+            data = f.read(1024 * 1024 * 30)
+            assert len(data) == 10 * 1000 * 1000 - (5 * 1024 * 1024)
+
+            f.seek(30 * 1024 * 1024)
+            data = f.read(1024 * 1024 * 30)
+            assert len(data) == 0
+
+    @pytest.mark.parametrize("piecefield_obj", [BigfilePiecefield, BigfilePiecefieldPacked])
+    def testPiecefield(self, piecefield_obj, site):
+        testdatas = [
+            b"\x01" * 100 + b"\x00" * 900 + b"\x01" * 4000 + b"\x00" * 4999 + b"\x01",
+            b"\x00\x01\x00\x01\x00\x01" * 10 + b"\x00\x01" * 90 + b"\x01\x00" * 400 + b"\x00" * 4999,
+            b"\x01" * 10000,
+            b"\x00" * 10000
+        ]
+        for testdata in testdatas:
+            piecefield = piecefield_obj()
+
+            piecefield.frombytes(testdata)
+            assert piecefield.tobytes() == testdata
+            assert piecefield[0] == testdata[0]
+            assert piecefield[100] == testdata[100]
+            assert piecefield[1000] == testdata[1000]
+            assert piecefield[len(testdata) - 1] == testdata[len(testdata) - 1]
+
+            packed = piecefield.pack()
+            piecefield_new = piecefield_obj()
+            piecefield_new.unpack(packed)
+            assert piecefield.tobytes() == piecefield_new.tobytes()
+            assert piecefield_new.tobytes() == testdata
+
+    def testFileGet(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        site_temp.connection_server = FileServer(file_server.ip, 1545)
+        site_temp.connection_server.sites[site_temp.address] = site_temp
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        # Download second block
+        with site_temp.storage.openBigfile(inner_path) as f:
+            f.seek(1024 * 1024)
+            assert f.read(1024)[0:1] != b"\0"
+
+        # Make sure first block not download
+        with site_temp.storage.open(inner_path) as f:
+            assert f.read(1024)[0:1] == b"\0"
+
+        peer2 = site.addPeer(file_server.ip, 1545, return_peer=True)
+
+        # Should drop error on first block request
+        assert not peer2.getFile(site.address, "%s|0-%s" % (inner_path, 1024 * 1024 * 1))
+
+        # Should not drop error for second block request
+        assert peer2.getFile(site.address, "%s|%s-%s" % (inner_path, 1024 * 1024 * 1, 1024 * 1024 * 2))
+
+    def benchmarkPeerMemory(self, site, file_server):
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        import psutil, os
+        meminfo = psutil.Process(os.getpid()).memory_info
+
+        mem_s = meminfo()[0]
+        s = time.time()
+        for i in range(25000):
+            site.addPeer(file_server.ip, i)
+        print("%.3fs MEM: + %sKB" % (time.time() - s, (meminfo()[0] - mem_s) / 1024))  # 0.082s MEM: + 6800KB
+        print(list(site.peers.values())[0].piecefields)
+
+    def testUpdatePiecefield(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        server1 = file_server
+        server1.sites[site.address] = site
+        server2 = FileServer(file_server.ip, 1545)
+        server2.sites[site_temp.address] = site_temp
+        site_temp.connection_server = server2
+
+        # Add file_server as peer to client
+        server2_peer1 = site_temp.addPeer(file_server.ip, 1544)
+
+        # Testing piecefield sync
+        assert len(server2_peer1.piecefields) == 0
+        assert server2_peer1.updatePiecefields()  # Query piecefields from peer
+        assert len(server2_peer1.piecefields) > 0
+
+    def testWorkerManagerPiecefieldDeny(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        server1 = file_server
+        server1.sites[site.address] = site
+        server2 = FileServer(file_server.ip, 1545)
+        server2.sites[site_temp.address] = site_temp
+        site_temp.connection_server = server2
+
+        # Add file_server as peer to client
+        server2_peer1 = site_temp.addPeer(file_server.ip, 1544)  # Working
+
+        site_temp.downloadContent("content.json", download_files=False)
+        site_temp.needFile("data/optional.any.iso.piecemap.msgpack")
+
+        # Add fake peers with optional files downloaded
+        for i in range(5):
+            fake_peer = site_temp.addPeer("127.0.1.%s" % i, 1544)
+            fake_peer.hashfield = site.content_manager.hashfield
+            fake_peer.has_hashfield = True
+
+        with Spy.Spy(WorkerManager, "addWorker") as requests:
+            site_temp.needFile("%s|%s-%s" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024))
+            site_temp.needFile("%s|%s-%s" % (inner_path, 6 * 1024 * 1024, 7 * 1024 * 1024))
+
+        # It should only request parts from peer1 as the other peers does not have the requested parts in piecefields
+        assert len([request[1] for request in requests if request[1] != server2_peer1]) == 0
+
+    def testWorkerManagerPiecefieldDownload(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        server1 = file_server
+        server1.sites[site.address] = site
+        server2 = FileServer(file_server.ip, 1545)
+        server2.sites[site_temp.address] = site_temp
+        site_temp.connection_server = server2
+        sha512 = site.content_manager.getFileInfo(inner_path)["sha512"]
+
+        # Create 10 fake peer for each piece
+        for i in range(10):
+            peer = Peer(file_server.ip, 1544, site_temp, server2)
+            peer.piecefields[sha512][i] = b"\x01"
+            peer.updateHashfield = mock.MagicMock(return_value=False)
+            peer.updatePiecefields = mock.MagicMock(return_value=False)
+            peer.findHashIds = mock.MagicMock(return_value={"nope": []})
+            peer.hashfield = site.content_manager.hashfield
+            peer.has_hashfield = True
+            peer.key = "Peer:%s" % i
+            site_temp.peers["Peer:%s" % i] = peer
+
+        site_temp.downloadContent("content.json", download_files=False)
+        site_temp.needFile("data/optional.any.iso.piecemap.msgpack")
+
+        with Spy.Spy(Peer, "getFile") as requests:
+            for i in range(10):
+                site_temp.needFile("%s|%s-%s" % (inner_path, i * 1024 * 1024, (i + 1) * 1024 * 1024))
+
+        assert len(requests) == 10
+        for i in range(10):
+            assert requests[i][0] == site_temp.peers["Peer:%s" % i]  # Every part should be requested from piece owner peer
+
+    def testDownloadStats(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        client = ConnectionServer(file_server.ip, 1545)
+        site_temp.connection_server = client
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        # Open virtual file
+        assert not site_temp.storage.isFile(inner_path)
+
+        # Check size before downloads
+        assert site_temp.settings["size"] < 10 * 1024 * 1024
+        assert site_temp.settings["optional_downloaded"] == 0
+        size_piecemap = site_temp.content_manager.getFileInfo(inner_path + ".piecemap.msgpack")["size"]
+        size_bigfile = site_temp.content_manager.getFileInfo(inner_path)["size"]
+
+        with site_temp.storage.openBigfile(inner_path) as f:
+            assert b"\0" not in f.read(1024)
+            assert site_temp.settings["optional_downloaded"] == size_piecemap + size_bigfile
+
+        with site_temp.storage.openBigfile(inner_path) as f:
+            # Don't count twice
+            assert b"\0" not in f.read(1024)
+            assert site_temp.settings["optional_downloaded"] == size_piecemap + size_bigfile
+
+            # Add second block
+            assert b"\0" not in f.read(1024 * 1024)
+            assert site_temp.settings["optional_downloaded"] == size_piecemap + size_bigfile
+
+    def testPrebuffer(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        client = ConnectionServer(file_server.ip, 1545)
+        site_temp.connection_server = client
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        # Open virtual file
+        assert not site_temp.storage.isFile(inner_path)
+
+        with site_temp.storage.openBigfile(inner_path, prebuffer=1024 * 1024 * 2) as f:
+            with Spy.Spy(FileRequest, "route") as requests:
+                f.seek(5 * 1024 * 1024)
+                assert f.read(7) == b"Test524"
+            # assert len(requests) == 3  # 1x piecemap + 1x getpiecefield + 1x for pieces
+            assert len([task for task in site_temp.worker_manager.tasks if task["inner_path"].startswith(inner_path)]) == 2
+
+            time.sleep(0.5)  # Wait prebuffer download
+
+            sha512 = site.content_manager.getFileInfo(inner_path)["sha512"]
+            assert site_temp.storage.piecefields[sha512].tostring() == "0000011100"
+
+            # No prebuffer beyond end of the file
+            f.seek(9 * 1024 * 1024)
+            assert b"\0" not in f.read(7)
+
+            assert len([task for task in site_temp.worker_manager.tasks if task["inner_path"].startswith(inner_path)]) == 0
+
+    def testDownloadAllPieces(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        client = ConnectionServer(file_server.ip, 1545)
+        site_temp.connection_server = client
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        # Open virtual file
+        assert not site_temp.storage.isFile(inner_path)
+
+        with Spy.Spy(FileRequest, "route") as requests:
+            site_temp.needFile("%s|all" % inner_path)
+
+        assert len(requests) == 12  # piecemap.msgpack, getPiecefields, 10 x piece
+
+        # Don't re-download already got pieces
+        with Spy.Spy(FileRequest, "route") as requests:
+            site_temp.needFile("%s|all" % inner_path)
+
+        assert len(requests) == 0
+
+    def testFileSize(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        client = ConnectionServer(file_server.ip, 1545)
+        site_temp.connection_server = client
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        # Open virtual file
+        assert not site_temp.storage.isFile(inner_path)
+
+        # Download first block
+        site_temp.needFile("%s|%s-%s" % (inner_path, 0 * 1024 * 1024, 1 * 1024 * 1024))
+        assert site_temp.storage.getSize(inner_path) < 1000 * 1000 * 10  # Size on the disk should be smaller than the real size
+
+        site_temp.needFile("%s|%s-%s" % (inner_path, 9 * 1024 * 1024, 10 * 1024 * 1024))
+        assert site_temp.storage.getSize(inner_path) == site.storage.getSize(inner_path)
+
+    def testFileRename(self, file_server, site, site_temp):
+        inner_path = self.createBigfile(site)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        site_temp.connection_server = FileServer(file_server.ip, 1545)
+        site_temp.connection_server.sites[site_temp.address] = site_temp
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        with Spy.Spy(FileRequest, "route") as requests:
+            site_temp.needFile("%s|%s-%s" % (inner_path, 0, 1 * self.piece_size))
+
+        assert len([req for req in requests if req[1] == "streamFile"]) == 2  # 1 piece + piecemap
+
+        # Rename the file
+        inner_path_new = inner_path.replace(".iso", "-new.iso")
+        site.storage.rename(inner_path, inner_path_new)
+        site.storage.delete("data/optional.any.iso.piecemap.msgpack")
+        assert site.content_manager.sign("content.json", self.privatekey, remove_missing_optional=True)
+
+        files_optional = site.content_manager.contents["content.json"]["files_optional"].keys()
+
+        assert "data/optional.any-new.iso.piecemap.msgpack" in files_optional
+        assert "data/optional.any.iso.piecemap.msgpack" not in files_optional
+        assert "data/optional.any.iso" not in files_optional
+
+        with Spy.Spy(FileRequest, "route") as requests:
+            site.publish()
+            time.sleep(0.1)
+            site_temp.download(blind_includes=True).join(timeout=5)  # Wait for download
+
+            assert len([req[1] for req in requests if req[1] == "streamFile"]) == 0
+
+            with site_temp.storage.openBigfile(inner_path_new, prebuffer=0) as f:
+                f.read(1024)
+
+                # First piece already downloaded
+                assert [req for req in requests if req[1] == "streamFile"] == []
+
+                # Second piece needs to be downloaded + changed piecemap
+                f.seek(self.piece_size)
+                f.read(1024)
+                assert [req[3]["inner_path"] for req in requests if req[1] == "streamFile"] == [inner_path_new + ".piecemap.msgpack", inner_path_new]
+
+    @pytest.mark.parametrize("size", [1024 * 3, 1024 * 1024 * 3, 1024 * 1024 * 30])
+    def testNullFileRead(self, file_server, site, site_temp, size):
+        inner_path = "data/optional.iso"
+
+        f = site.storage.open(inner_path, "w")
+        f.write("\0" * size)
+        f.close()
+        assert site.content_manager.sign("content.json", self.privatekey)
+
+        # Init source server
+        site.connection_server = file_server
+        file_server.sites[site.address] = site
+
+        # Init client server
+        site_temp.connection_server = FileServer(file_server.ip, 1545)
+        site_temp.connection_server.sites[site_temp.address] = site_temp
+        site_temp.addPeer(file_server.ip, 1544)
+
+        # Download site
+        site_temp.download(blind_includes=True).join(timeout=5)
+
+        if "piecemap" in site.content_manager.getFileInfo(inner_path):  # Bigfile
+            site_temp.needFile(inner_path + "|all")
+        else:
+            site_temp.needFile(inner_path)
+
+
+        assert site_temp.storage.getSize(inner_path) == size

plugins/Bigfile/Test/conftest.py

@@ -0,0 +1 @@
+from src.Test.conftest import *

plugins/Bigfile/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/Bigfile/__init__.py

@@ -0,0 +1,2 @@
+from . import BigfilePlugin
+from .BigfilePiecefield import BigfilePiecefield, BigfilePiecefieldPacked

plugins/Chart/ChartCollector.py

@@ -0,0 +1,181 @@
+import time
+import sys
+import collections
+import itertools
+import logging
+
+import gevent
+from util import helper
+from Config import config
+
+
+class ChartCollector(object):
+    def __init__(self, db):
+        self.db = db
+        if config.action == "main":
+            gevent.spawn_later(60 * 3, self.collector)
+        self.log = logging.getLogger("ChartCollector")
+        self.last_values = collections.defaultdict(dict)
+
+    def setInitialLastValues(self, sites):
+        # Recover last value of site bytes/sent
+        for site in sites:
+            self.last_values["site:" + site.address]["site_bytes_recv"] = site.settings.get("bytes_recv", 0)
+            self.last_values["site:" + site.address]["site_bytes_sent"] = site.settings.get("bytes_sent", 0)
+
+    def getCollectors(self):
+        collectors = {}
+        import main
+        file_server = main.file_server
+        sites = file_server.sites
+        if not sites:
+            return collectors
+        content_db = list(sites.values())[0].content_manager.contents.db
+
+        # Connection stats
+        collectors["connection"] = lambda: len(file_server.connections)
+        collectors["connection_in"] = (
+            lambda: len([1 for connection in file_server.connections if connection.type == "in"])
+        )
+        collectors["connection_onion"] = (
+            lambda: len([1 for connection in file_server.connections if connection.ip.endswith(".onion")])
+        )
+        collectors["connection_ping_avg"] = (
+            lambda: round(1000 * helper.avg(
+                [connection.last_ping_delay for connection in file_server.connections if connection.last_ping_delay]
+            ))
+        )
+        collectors["connection_ping_min"] = (
+            lambda: round(1000 * min(
+                [connection.last_ping_delay for connection in file_server.connections if connection.last_ping_delay]
+            ))
+        )
+        collectors["connection_rev_avg"] = (
+            lambda: helper.avg(
+                [connection.handshake["rev"] for connection in file_server.connections if connection.handshake]
+            )
+        )
+
+        # Request stats
+        collectors["file_bytes_recv|change"] = lambda: file_server.bytes_recv
+        collectors["file_bytes_sent|change"] = lambda: file_server.bytes_sent
+        collectors["request_num_recv|change"] = lambda: file_server.num_recv
+        collectors["request_num_sent|change"] = lambda: file_server.num_sent
+
+        # Limit
+        collectors["optional_limit"] = lambda: content_db.getOptionalLimitBytes()
+        collectors["optional_used"] = lambda: content_db.getOptionalUsedBytes()
+        collectors["optional_downloaded"] = lambda: sum([site.settings.get("optional_downloaded", 0) for site in sites.values()])
+
+        # Peers
+        collectors["peer"] = lambda peers: len(peers)
+        collectors["peer_onion"] = lambda peers: len([True for peer in peers if ".onion" in peer])
+
+        # Size
+        collectors["size"] = lambda: sum([site.settings.get("size", 0) for site in sites.values()])
+        collectors["size_optional"] = lambda: sum([site.settings.get("size_optional", 0) for site in sites.values()])
+        collectors["content"] = lambda: sum([len(site.content_manager.contents) for site in sites.values()])
+
+        return collectors
+
+    def getSiteCollectors(self):
+        site_collectors = {}
+
+        # Size
+        site_collectors["site_size"] = lambda site: site.settings.get("size", 0)
+        site_collectors["site_size_optional"] = lambda site: site.settings.get("size_optional", 0)
+        site_collectors["site_optional_downloaded"] = lambda site: site.settings.get("optional_downloaded", 0)
+        site_collectors["site_content"] = lambda site: len(site.content_manager.contents)
+
+        # Data transfer
+        site_collectors["site_bytes_recv|change"] = lambda site: site.settings.get("bytes_recv", 0)
+        site_collectors["site_bytes_sent|change"] = lambda site: site.settings.get("bytes_sent", 0)
+
+        # Peers
+        site_collectors["site_peer"] = lambda site: len(site.peers)
+        site_collectors["site_peer_onion"] = lambda site: len(
+            [True for peer in site.peers.values() if peer.ip.endswith(".onion")]
+        )
+        site_collectors["site_peer_connected"] = lambda site: len([True for peer in site.peers.values() if peer.connection])
+
+        return site_collectors
+
+    def getUniquePeers(self):
+        import main
+        sites = main.file_server.sites
+        return set(itertools.chain.from_iterable(
+            [site.peers.keys() for site in sites.values()]
+        ))
+
+    def collectDatas(self, collectors, last_values, site=None):
+        if site is None:
+            peers = self.getUniquePeers()
+        datas = {}
+        for key, collector in collectors.items():
+            try:
+                if site:
+                    value = collector(site)
+                elif key.startswith("peer"):
+                    value = collector(peers)
+                else:
+                    value = collector()
+            except Exception as err:
+                self.log.info("Collector %s error: %s" % (key, err))
+                value = None
+
+            if "|change" in key:  # Store changes relative to last value
+                key = key.replace("|change", "")
+                last_value = last_values.get(key, 0)
+                last_values[key] = value
+                value = value - last_value
+
+            if value is None:
+                datas[key] = None
+            else:
+                datas[key] = round(value, 3)
+        return datas
+
+    def collectGlobal(self, collectors, last_values):
+        now = int(time.time())
+        s = time.time()
+        datas = self.collectDatas(collectors, last_values["global"])
+        values = []
+        for key, value in datas.items():
+            values.append((self.db.getTypeId(key), value, now))
+        self.log.debug("Global collectors done in %.3fs" % (time.time() - s))
+
+        s = time.time()
+        cur = self.db.getCursor()
+        cur.cursor.executemany("INSERT INTO data (type_id, value, date_added) VALUES (?, ?, ?)", values)
+        cur.close()
+        self.log.debug("Global collectors inserted in %.3fs" % (time.time() - s))
+
+    def collectSites(self, sites, collectors, last_values):
+        now = int(time.time())
+        s = time.time()
+        values = []
+        for address, site in sites.items():
+            site_datas = self.collectDatas(collectors, last_values["site:%s" % address], site)
+            for key, value in site_datas.items():
+                values.append((self.db.getTypeId(key), self.db.getSiteId(address), value, now))
+            time.sleep(0.001)
+        self.log.debug("Site collections done in %.3fs" % (time.time() - s))
+
+        s = time.time()
+        cur = self.db.getCursor()
+        cur.cursor.executemany("INSERT INTO data (type_id, site_id, value, date_added) VALUES (?, ?, ?, ?)", values)
+        cur.close()
+        self.log.debug("Site collectors inserted in %.3fs" % (time.time() - s))
+
+    def collector(self):
+        collectors = self.getCollectors()
+        site_collectors = self.getSiteCollectors()
+        import main
+        sites = main.file_server.sites
+        i = 0
+        while 1:
+            self.collectGlobal(collectors, self.last_values)
+            if i % 12 == 0:  # Only collect sites data every hour
+                self.collectSites(sites, site_collectors, self.last_values)
+            time.sleep(60 * 5)
+            i += 1

plugins/Chart/ChartDb.py

@@ -0,0 +1,133 @@
+from Config import config
+from Db.Db import Db
+import time
+
+
+class ChartDb(Db):
+    def __init__(self):
+        self.version = 2
+        super(ChartDb, self).__init__(self.getSchema(), "%s/chart.db" % config.data_dir)
+        self.foreign_keys = True
+        self.checkTables()
+        self.sites = self.loadSites()
+        self.types = self.loadTypes()
+
+    def getSchema(self):
+        schema = {}
+        schema["db_name"] = "Chart"
+        schema["tables"] = {}
+        schema["tables"]["data"] = {
+            "cols": [
+                ["data_id", "INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE"],
+                ["type_id", "INTEGER NOT NULL"],
+                ["site_id", "INTEGER"],
+                ["value", "INTEGER"],
+                ["date_added", "DATETIME DEFAULT (CURRENT_TIMESTAMP)"]
+            ],
+            "indexes": [
+                "CREATE INDEX site_id ON data (site_id)",
+                "CREATE INDEX date_added ON data (date_added)"
+            ],
+            "schema_changed": 2
+        }
+        schema["tables"]["type"] = {
+            "cols": [
+                ["type_id", "INTEGER PRIMARY KEY NOT NULL UNIQUE"],
+                ["name", "TEXT"]
+            ],
+            "schema_changed": 1
+        }
+        schema["tables"]["site"] = {
+            "cols": [
+                ["site_id", "INTEGER PRIMARY KEY NOT NULL UNIQUE"],
+                ["address", "TEXT"]
+            ],
+            "schema_changed": 1
+        }
+        return schema
+
+    def getTypeId(self, name):
+        if name not in self.types:
+            self.execute("INSERT INTO type ?", {"name": name})
+            self.types[name] = self.cur.cursor.lastrowid
+
+        return self.types[name]
+
+    def getSiteId(self, address):
+        if address not in self.sites:
+            self.execute("INSERT INTO site ?", {"address": address})
+            self.sites[address] = self.cur.cursor.lastrowid
+
+        return self.sites[address]
+
+    def loadSites(self):
+        sites = {}
+        for row in self.execute("SELECT * FROM site"):
+            sites[row["address"]] = row["site_id"]
+        return sites
+
+    def loadTypes(self):
+        types = {}
+        for row in self.execute("SELECT * FROM type"):
+            types[row["name"]] = row["type_id"]
+        return types
+
+    def deleteSite(self, address):
+        if address in self.sites:
+            site_id = self.sites[address]
+            del self.sites[address]
+            self.execute("DELETE FROM site WHERE ?", {"site_id": site_id})
+            self.execute("DELETE FROM data WHERE ?", {"site_id": site_id})
+
+    def archive(self):
+        week_back = 1
+        while 1:
+            s = time.time()
+            date_added_from = time.time() - 60 * 60 * 24 * 7 * (week_back + 1)
+            date_added_to = date_added_from + 60 * 60 * 24 * 7
+            res = self.execute("""
+                SELECT
+                 MAX(date_added) AS date_added,
+                 SUM(value) AS value,
+                 GROUP_CONCAT(data_id) AS data_ids,
+                 type_id,
+                 site_id,
+                 COUNT(*) AS num
+                FROM data
+                WHERE
+                 site_id IS NULL AND
+                 date_added > :date_added_from AND
+                 date_added < :date_added_to
+                GROUP BY strftime('%Y-%m-%d %H', date_added, 'unixepoch', 'localtime'), type_id
+            """, {"date_added_from": date_added_from, "date_added_to": date_added_to})
+
+            num_archived = 0
+            cur = self.getCursor()
+            for row in res:
+                if row["num"] == 1:
+                    continue
+                cur.execute("INSERT INTO data ?", {
+                    "type_id": row["type_id"],
+                    "site_id": row["site_id"],
+                    "value": row["value"],
+                    "date_added": row["date_added"]
+                })
+                cur.execute("DELETE FROM data WHERE data_id IN (%s)" % row["data_ids"])
+                num_archived += row["num"]
+            self.log.debug("Archived %s data from %s weeks ago in %.3fs" % (num_archived, week_back, time.time() - s))
+            week_back += 1
+            time.sleep(0.1)
+            if num_archived == 0:
+                break
+        # Only keep 6 month of global stats
+        self.execute(
+            "DELETE FROM data WHERE site_id IS NULL AND date_added < :date_added_limit",
+            {"date_added_limit": time.time() - 60 * 60 * 24 * 30 * 6 }
+        )
+        # Only keep 1 month of site stats
+        self.execute(
+            "DELETE FROM data WHERE site_id IS NOT NULL AND date_added < :date_added_limit",
+            {"date_added_limit": time.time() - 60 * 60 * 24 * 30 }
+        )
+        if week_back > 1:
+            self.execute("VACUUM")

plugins/Chart/ChartPlugin.py

@@ -0,0 +1,60 @@
+import time
+import itertools
+
+import gevent
+
+from Config import config
+from util import helper
+from Plugin import PluginManager
+from .ChartDb import ChartDb
+from .ChartCollector import ChartCollector
+
+if "db" not in locals().keys():  # Share on reloads
+    db = ChartDb()
+    gevent.spawn_later(10 * 60, db.archive)
+    helper.timer(60 * 60 * 6, db.archive)
+    collector = ChartCollector(db)
+
+@PluginManager.registerTo("SiteManager")
+class SiteManagerPlugin(object):
+    def load(self, *args, **kwargs):
+        back = super(SiteManagerPlugin, self).load(*args, **kwargs)
+        collector.setInitialLastValues(self.sites.values())
+        return back
+
+    def delete(self, address, *args, **kwargs):
+        db.deleteSite(address)
+        return super(SiteManagerPlugin, self).delete(address, *args, **kwargs)
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def actionChartDbQuery(self, to, query, params=None):
+        if not "ADMIN" in self.permissions:
+            return {"error": "No permission"}
+
+        if config.debug or config.verbose:
+            s = time.time()
+        rows = []
+        try:
+            if not query.strip().upper().startswith("SELECT"):
+                raise Exception("Only SELECT query supported")
+            res = db.execute(query, params)
+        except Exception as err:  # Response the error to client
+            self.log.error("ChartDbQuery error: %s" % err)
+            return {"error": str(err)}
+        # Convert result to dict
+        for row in res:
+            rows.append(dict(row))
+        if config.verbose and time.time() - s > 0.1:  # Log slow query
+            self.log.debug("Slow query: %s (%.3fs)" % (query, time.time() - s))
+        return rows
+
+    def actionChartGetPeerLocations(self, to):
+        if not "ADMIN" in self.permissions:
+            return {"error": "No permission"}
+
+        peers = {}
+        for site in self.server.sites.values():
+            peers.update(site.peers)
+        peer_locations = self.getPeerLocations(peers)
+        return peer_locations

plugins/Chart/__init__.py

@@ -0,0 +1 @@
+from . import ChartPlugin

plugins/ContentFilter/ContentFilterPlugin.py

@@ -0,0 +1,216 @@
+import time
+import re
+import html
+import hashlib
+
+from Plugin import PluginManager
+from Translate import Translate
+from Config import config
+
+from .ContentFilterStorage import ContentFilterStorage
+
+
+if "_" not in locals():
+    _ = Translate("plugins/ContentFilter/languages/")
+
+
+@PluginManager.registerTo("SiteManager")
+class SiteManagerPlugin(object):
+    def load(self, *args, **kwargs):
+        global filter_storage
+        super(SiteManagerPlugin, self).load(*args, **kwargs)
+        filter_storage = ContentFilterStorage(site_manager=self)
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    # Mute
+    def cbMuteAdd(self, to, auth_address, cert_user_id, reason):
+        filter_storage.file_content["mutes"][auth_address] = {
+            "cert_user_id": cert_user_id, "reason": reason, "source": self.site.address, "date_added": time.time()
+        }
+        filter_storage.save()
+        filter_storage.changeDbs(auth_address, "remove")
+        self.response(to, "ok")
+
+    def actionMuteAdd(self, to, auth_address, cert_user_id, reason):
+        if "ADMIN" in self.getPermissions(to):
+            self.cbMuteAdd(to, auth_address, cert_user_id, reason)
+        else:
+            self.cmd(
+                "confirm",
+                [_["Hide all content from <b>%s</b>?"] % html.escape(cert_user_id), _["Mute"]],
+                lambda res: self.cbMuteAdd(to, auth_address, cert_user_id, reason)
+            )
+
+    def cbMuteRemove(self, to, auth_address):
+        del filter_storage.file_content["mutes"][auth_address]
+        filter_storage.save()
+        filter_storage.changeDbs(auth_address, "load")
+        self.response(to, "ok")
+
+    def actionMuteRemove(self, to, auth_address):
+        if "ADMIN" in self.getPermissions(to):
+            self.cbMuteRemove(to, auth_address)
+        else:
+            self.cmd(
+                "confirm",
+                [_["Unmute <b>%s</b>?"] % html.escape(filter_storage.file_content["mutes"][auth_address]["cert_user_id"]), _["Unmute"]],
+                lambda res: self.cbMuteRemove(to, auth_address)
+            )
+
+    def actionMuteList(self, to):
+        if "ADMIN" in self.getPermissions(to):
+            self.response(to, filter_storage.file_content["mutes"])
+        else:
+            return self.response(to, {"error": "Forbidden: Only ADMIN sites can list mutes"})
+
+    # Siteblock
+    def actionSiteblockAdd(self, to, site_address, reason=None):
+        if "ADMIN" not in self.getPermissions(to):
+            return self.response(to, {"error": "Forbidden: Only ADMIN sites can add to blocklist"})
+        filter_storage.file_content["siteblocks"][site_address] = {"date_added": time.time(), "reason": reason}
+        filter_storage.save()
+        self.response(to, "ok")
+
+    def actionSiteblockRemove(self, to, site_address):
+        if "ADMIN" not in self.getPermissions(to):
+            return self.response(to, {"error": "Forbidden: Only ADMIN sites can remove from blocklist"})
+        del filter_storage.file_content["siteblocks"][site_address]
+        filter_storage.save()
+        self.response(to, "ok")
+
+    def actionSiteblockList(self, to):
+        if "ADMIN" in self.getPermissions(to):
+            self.response(to, filter_storage.file_content["siteblocks"])
+        else:
+            return self.response(to, {"error": "Forbidden: Only ADMIN sites can list blocklists"})
+
+    # Include
+    def actionFilterIncludeAdd(self, to, inner_path, description=None, address=None):
+        if address:
+            if "ADMIN" not in self.getPermissions(to):
+                return self.response(to, {"error": "Forbidden: Only ADMIN sites can manage different site include"})
+            site = self.server.sites[address]
+        else:
+            address = self.site.address
+            site = self.site
+
+        if "ADMIN" in self.getPermissions(to):
+            self.cbFilterIncludeAdd(to, True, address, inner_path, description)
+        else:
+            content = site.storage.loadJson(inner_path)
+            title = _["New shared global content filter: <b>%s</b> (%s sites, %s users)"] % (
+                html.escape(inner_path), len(content.get("siteblocks", {})), len(content.get("mutes", {}))
+            )
+
+            self.cmd(
+                "confirm",
+                [title, "Add"],
+                lambda res: self.cbFilterIncludeAdd(to, res, address, inner_path, description)
+            )
+
+    def cbFilterIncludeAdd(self, to, res, address, inner_path, description):
+        if not res:
+            self.response(to, res)
+            return False
+
+        filter_storage.includeAdd(address, inner_path, description)
+        self.response(to, "ok")
+
+    def actionFilterIncludeRemove(self, to, inner_path, address=None):
+        if address:
+            if "ADMIN" not in self.getPermissions(to):
+                return self.response(to, {"error": "Forbidden: Only ADMIN sites can manage different site include"})
+        else:
+            address = self.site.address
+
+        key = "%s/%s" % (address, inner_path)
+        if key not in filter_storage.file_content["includes"]:
+            self.response(to, {"error": "Include not found"})
+        filter_storage.includeRemove(address, inner_path)
+        self.response(to, "ok")
+
+    def actionFilterIncludeList(self, to, all_sites=False, filters=False):
+        if all_sites and "ADMIN" not in self.getPermissions(to):
+            return self.response(to, {"error": "Forbidden: Only ADMIN sites can list all sites includes"})
+
+        back = []
+        includes = filter_storage.file_content.get("includes", {}).values()
+        for include in includes:
+            if not all_sites and include["address"] != self.site.address:
+                continue
+            if filters:
+                include = dict(include)  # Don't modify original file_content
+                include_site = filter_storage.site_manager.get(include["address"])
+                if not include_site:
+                    continue
+                content = include_site.storage.loadJson(include["inner_path"])
+                include["mutes"] = content.get("mutes", {})
+                include["siteblocks"] = content.get("siteblocks", {})
+            back.append(include)
+        self.response(to, back)
+
+
+@PluginManager.registerTo("SiteStorage")
+class SiteStoragePlugin(object):
+    def updateDbFile(self, inner_path, file=None, cur=None):
+        if file is not False:  # File deletion always allowed
+            # Find for bitcoin addresses in file path
+            matches = re.findall("/(1[A-Za-z0-9]{26,35})/", inner_path)
+            # Check if any of the adresses are in the mute list
+            for auth_address in matches:
+                if filter_storage.isMuted(auth_address):
+                    self.log.debug("Mute match: %s, ignoring %s" % (auth_address, inner_path))
+                    return False
+
+        return super(SiteStoragePlugin, self).updateDbFile(inner_path, file=file, cur=cur)
+
+    def onUpdated(self, inner_path, file=None):
+        file_path = "%s/%s" % (self.site.address, inner_path)
+        if file_path in filter_storage.file_content["includes"]:
+            self.log.debug("Filter file updated: %s" % inner_path)
+            filter_storage.includeUpdateAll()
+        return super(SiteStoragePlugin, self).onUpdated(inner_path, file=file)
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    def actionWrapper(self, path, extra_headers=None):
+        match = re.match("/(?P<address>[A-Za-z0-9\._-]+)(?P<inner_path>/.*|$)", path)
+        if not match:
+            return False
+        address = match.group("address")
+
+        if self.server.site_manager.get(address):  # Site already exists
+            return super(UiRequestPlugin, self).actionWrapper(path, extra_headers)
+
+        if self.server.site_manager.isDomain(address):
+            address = self.server.site_manager.resolveDomain(address)
+
+        if address:
+            address_sha256 = "0x" + hashlib.sha256(address.encode("utf8")).hexdigest()
+        else:
+            address_sha256 = None
+
+        if filter_storage.isSiteblocked(address) or filter_storage.isSiteblocked(address_sha256):
+            site = self.server.site_manager.get(config.homepage)
+            if not extra_headers:
+                extra_headers = {}
+
+            script_nonce = self.getScriptNonce()
+
+            self.sendHeader(extra_headers=extra_headers, script_nonce=script_nonce)
+            return iter([super(UiRequestPlugin, self).renderWrapper(
+                site, path, "uimedia/plugins/contentfilter/blocklisted.html?address=" + address,
+                "Blacklisted site", extra_headers, show_loadingscreen=False, script_nonce=script_nonce
+            )])
+        else:
+            return super(UiRequestPlugin, self).actionWrapper(path, extra_headers)
+
+    def actionUiMedia(self, path, *args, **kwargs):
+        if path.startswith("/uimedia/plugins/contentfilter/"):
+            file_path = path.replace("/uimedia/plugins/contentfilter/", "plugins/ContentFilter/media/")
+            return self.actionFile(file_path)
+        else:
+            return super(UiRequestPlugin, self).actionUiMedia(path)

plugins/ContentFilter/ContentFilterStorage.py

@@ -0,0 +1,140 @@
+import os
+import json
+import logging
+import collections
+import time
+
+from Debug import Debug
+from Plugin import PluginManager
+from Config import config
+from util import helper
+
+class ContentFilterStorage(object):
+    def __init__(self, site_manager):
+        self.log = logging.getLogger("ContentFilterStorage")
+        self.file_path = "%s/filters.json" % config.data_dir
+        self.site_manager = site_manager
+        self.file_content = self.load()
+
+        # Set default values for filters.json
+        if not self.file_content:
+            self.file_content = {}
+
+        # Site blacklist renamed to site blocks
+        if "site_blacklist" in self.file_content:
+            self.file_content["siteblocks"] = self.file_content["site_blacklist"]
+            del self.file_content["site_blacklist"]
+
+        for key in ["mutes", "siteblocks", "includes"]:
+            if key not in self.file_content:
+                self.file_content[key] = {}
+
+        self.include_filters = collections.defaultdict(set)  # Merged list of mutes and blacklists from all include
+        self.includeUpdateAll(update_site_dbs=False)
+
+    def load(self):
+        # Rename previously used mutes.json -> filters.json
+        if os.path.isfile("%s/mutes.json" % config.data_dir):
+            self.log.info("Renaming mutes.json to filters.json...")
+            os.rename("%s/mutes.json" % config.data_dir, self.file_path)
+        if os.path.isfile(self.file_path):
+            try:
+                return json.load(open(self.file_path))
+            except Exception as err:
+                self.log.error("Error loading filters.json: %s" % err)
+                return None
+        else:
+            return None
+
+    def includeUpdateAll(self, update_site_dbs=True):
+        s = time.time()
+        new_include_filters = collections.defaultdict(set)
+
+        # Load all include files data into a merged set
+        for include_path in self.file_content["includes"]:
+            address, inner_path = include_path.split("/", 1)
+            try:
+                content = self.site_manager.get(address).storage.loadJson(inner_path)
+            except Exception as err:
+                self.log.warning(
+                    "Error loading include %s: %s" %
+                    (include_path, Debug.formatException(err))
+                )
+                continue
+
+            for key, val in content.items():
+                if type(val) is not dict:
+                    continue
+
+                new_include_filters[key].update(val.keys())
+
+        mutes_added = new_include_filters["mutes"].difference(self.include_filters["mutes"])
+        mutes_removed = self.include_filters["mutes"].difference(new_include_filters["mutes"])
+
+        self.include_filters = new_include_filters
+
+        if update_site_dbs:
+            for auth_address in mutes_added:
+                self.changeDbs(auth_address, "remove")
+
+            for auth_address in mutes_removed:
+                if not self.isMuted(auth_address):
+                    self.changeDbs(auth_address, "load")
+
+        num_mutes = len(self.include_filters["mutes"])
+        num_siteblocks = len(self.include_filters["siteblocks"])
+        self.log.debug(
+            "Loaded %s mutes, %s blocked sites from %s includes in %.3fs" %
+            (num_mutes, num_siteblocks, len(self.file_content["includes"]), time.time() - s)
+        )
+
+    def includeAdd(self, address, inner_path, description=None):
+        self.file_content["includes"]["%s/%s" % (address, inner_path)] = {
+            "date_added": time.time(),
+            "address": address,
+            "description": description,
+            "inner_path": inner_path
+        }
+        self.includeUpdateAll()
+        self.save()
+
+    def includeRemove(self, address, inner_path):
+        del self.file_content["includes"]["%s/%s" % (address, inner_path)]
+        self.includeUpdateAll()
+        self.save()
+
+    def save(self):
+        s = time.time()
+        helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8"))
+        self.log.debug("Saved in %.3fs" % (time.time() - s))
+
+    def isMuted(self, auth_address):
+        if auth_address in self.file_content["mutes"] or auth_address in self.include_filters["mutes"]:
+            return True
+        else:
+            return False
+
+    def isSiteblocked(self, address):
+        if address in self.file_content["siteblocks"] or address in self.include_filters["siteblocks"]:
+            return True
+        else:
+            return False
+
+    # Search and remove or readd files of an user
+    def changeDbs(self, auth_address, action):
+        self.log.debug("Mute action %s on user %s" % (action, auth_address))
+        res = list(self.site_manager.list().values())[0].content_manager.contents.db.execute(
+            "SELECT * FROM content LEFT JOIN site USING (site_id) WHERE inner_path LIKE :inner_path",
+            {"inner_path": "%%/%s/%%" % auth_address}
+        )
+        for row in res:
+            site = self.site_manager.sites.get(row["address"])
+            if not site:
+                continue
+            dir_inner_path = helper.getDirname(row["inner_path"])
+            for file_name in site.storage.walk(dir_inner_path):
+                if action == "remove":
+                    site.storage.onUpdated(dir_inner_path + file_name, False)
+                else:
+                    site.storage.onUpdated(dir_inner_path + file_name)
+                site.onFileDone(dir_inner_path + file_name)

plugins/ContentFilter/Test/TestContentFilter.py

@@ -0,0 +1,82 @@
+import pytest
+from ContentFilter import ContentFilterPlugin
+from Site import SiteManager
+
+
+@pytest.fixture
+def filter_storage():
+    ContentFilterPlugin.filter_storage = ContentFilterPlugin.ContentFilterStorage(SiteManager.site_manager)
+    return ContentFilterPlugin.filter_storage
+
+
+@pytest.mark.usefixtures("resetSettings")
+@pytest.mark.usefixtures("resetTempSettings")
+class TestContentFilter:
+    def createInclude(self, site):
+        site.storage.writeJson("filters.json", {
+            "mutes": {"1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C": {}},
+            "siteblocks": {site.address: {}}
+        })
+
+    def testIncludeLoad(self, site, filter_storage):
+        self.createInclude(site)
+        filter_storage.file_content["includes"]["%s/%s" % (site.address, "filters.json")] = {
+            "date_added": 1528295893,
+        }
+
+        assert not filter_storage.include_filters["mutes"]
+        assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+        assert not filter_storage.isSiteblocked(site.address)
+        filter_storage.includeUpdateAll(update_site_dbs=False)
+        assert len(filter_storage.include_filters["mutes"]) == 1
+        assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+        assert filter_storage.isSiteblocked(site.address)
+
+    def testIncludeAdd(self, site, filter_storage):
+        self.createInclude(site)
+        query_num_json = "SELECT COUNT(*) AS num FROM json WHERE directory = 'users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C'"
+        assert not filter_storage.isSiteblocked(site.address)
+        assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+        assert site.storage.query(query_num_json).fetchone()["num"] == 2
+
+        # Add include
+        filter_storage.includeAdd(site.address, "filters.json")
+
+        assert filter_storage.isSiteblocked(site.address)
+        assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+        assert site.storage.query(query_num_json).fetchone()["num"] == 0
+
+        # Remove include
+        filter_storage.includeRemove(site.address, "filters.json")
+
+        assert not filter_storage.isSiteblocked(site.address)
+        assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+        assert site.storage.query(query_num_json).fetchone()["num"] == 2
+
+    def testIncludeChange(self, site, filter_storage):
+        self.createInclude(site)
+        filter_storage.includeAdd(site.address, "filters.json")
+        assert filter_storage.isSiteblocked(site.address)
+        assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+
+        # Add new blocked site
+        assert not filter_storage.isSiteblocked("1Hello")
+
+        filter_content = site.storage.loadJson("filters.json")
+        filter_content["siteblocks"]["1Hello"] = {}
+        site.storage.writeJson("filters.json", filter_content)
+
+        assert filter_storage.isSiteblocked("1Hello")
+
+        # Add new muted user
+        query_num_json = "SELECT COUNT(*) AS num FROM json WHERE directory = 'users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q'"
+        assert not filter_storage.isMuted("1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q")
+        assert site.storage.query(query_num_json).fetchone()["num"] == 2
+
+        filter_content["mutes"]["1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q"] = {}
+        site.storage.writeJson("filters.json", filter_content)
+
+        assert filter_storage.isMuted("1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q")
+        assert site.storage.query(query_num_json).fetchone()["num"] == 0
+
+

plugins/ContentFilter/Test/conftest.py

@@ -0,0 +1 @@
+from src.Test.conftest import *

plugins/ContentFilter/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/ContentFilter/__init__.py

@@ -0,0 +1 @@
+from . import ContentFilterPlugin

plugins/ContentFilter/languages/hu.json

@@ -0,0 +1,6 @@
+{
+	"Hide all content from <b>%s</b>?": "<b>%s</b> tartalmaniak elrejtése?",
+	"Mute": "Elnémítás",
+	"Unmute <b>%s</b>?": "<b>%s</b> tartalmaniak megjelenítése?",
+	"Unmute": "Némítás visszavonása"
+}

plugins/ContentFilter/languages/it.json

@@ -0,0 +1,6 @@
+{
+	"Hide all content from <b>%s</b>?": "<b>%s</b> Vuoi nascondere i contenuti di questo utente ?",
+	"Mute": "Attiva Silenzia",
+	"Unmute <b>%s</b>?": "<b>%s</b> Vuoi mostrare i contenuti di questo utente ?",
+	"Unmute": "Disattiva Silenzia"
+}

plugins/ContentFilter/languages/pt-br.json

@@ -0,0 +1,6 @@
+{
+	"Hide all content from <b>%s</b>?": "<b>%s</b> Ocultar todo o conteúdo de ?",
+	"Mute": "Ativar o Silêncio",
+	"Unmute <b>%s</b>?": "<b>%s</b> Você quer mostrar o conteúdo deste usuário ?",
+	"Unmute": "Desligar o silêncio"
+}

plugins/ContentFilter/languages/zh-tw.json

@@ -0,0 +1,6 @@
+{
+	"Hide all content from <b>%s</b>?": "屏蔽 <b>%s</b> 的所有內容？",
+	"Mute": "屏蔽",
+	"Unmute <b>%s</b>?": "對 <b>%s</b> 解除屏蔽？",
+	"Unmute": "解除屏蔽"
+}

plugins/ContentFilter/languages/zh.json

@@ -0,0 +1,6 @@
+{
+	"Hide all content from <b>%s</b>?": "屏蔽 <b>%s</b> 的所有内容？",
+	"Mute": "屏蔽",
+	"Unmute <b>%s</b>?": "对 <b>%s</b> 解除屏蔽？",
+	"Unmute": "解除屏蔽"
+}

plugins/ContentFilter/media/blocklisted.html

@@ -0,0 +1,107 @@
+<html>
+<body>
+
+<style>
+.content { line-height: 24px; font-family: monospace; font-size: 14px; color: #636363; text-transform: uppercase; top: 38%; position: relative; text-align: center; perspective: 1000px }
+.content h1, .content h2 { font-weight: normal; letter-spacing: 1px; }
+.content h2 { font-size: 15px; }
+.content #details {
+    text-align: left; display: inline-block; width: 350px; background-color: white; padding: 17px 27px; border-radius: 0px;
+    box-shadow: 0px 2px 7px -1px #d8d8d8; text-transform: none; margin: 15px; transform: scale(0) rotateX(90deg); transition: all 0.6s cubic-bezier(0.785, 0.135, 0.15, 0.86);
+}
+.content #details #added { font-size: 12px; text-align: right; color: #a9a9a9; }
+
+#button { transition: all 1s cubic-bezier(0.075, 0.82, 0.165, 1); opacity: 0; transform: translateY(50px); transition-delay: 0.5s }
+.button {
+    padding: 8px 20px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; border-radius: 2px;
+    text-decoration: none; transition: all 0.5s; background-position: left center; display: inline-block; margin-top: 10px; color: black;
+}
+.button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; transition: none; }
+.button:active { position: relative; top: 1px; }
+.button:focus { outline: none; }
+
+.textbutton { color: #999; margin-top: 25px; display: inline-block; text-transform: none; font-family: Arial, Helvetica; text-decoration: none; padding: 5px 15px; }
+.textbutton-main { background-color: #FFF; color: #333; border-radius: 5px; }
+.textbutton:hover { text-decoration: underline; color: #333; transition: none !important; }
+.textbutton:active { background-color: #fafbfc; }
+</style>
+
+<div class="content">
+ <h1>Site blocked</h1>
+ <h2>This site is on your blocklist:</h2>
+ <div id="details">
+  <div id="reason">Too much image</div>
+  <div id="added">on 2015-01-25 12:32:11</div>
+ </div>
+ <div id="buttons">
+  <a href="/" class="textbutton textbutton-main" id="back">Back to homepage</a>
+  <a href="#Visit+Site" class="textbutton" id="visit">Remove from blocklist and visit the site</a>
+ </div>
+</div>
+
+<script type="text/javascript" src="js/ZeroFrame.js"></script>
+
+<script>
+function buf2hex(buffer) {
+    return Array.prototype.map.call(new Uint8Array(buffer), x => ('00' + x.toString(16)).slice(-2)).join('');
+}
+
+async function sha256hex(s) {
+    var buff = new TextEncoder("utf-8").encode(s)
+    return "0x" + buf2hex(await crypto.subtle.digest("SHA-256", buff))
+}
+
+class Page extends ZeroFrame {
+    onOpenWebsocket () {
+    	this.cmd("wrapperSetTitle", "Visiting a blocked site - ZeroNet")
+        this.cmd("siteInfo", {}, (site_info) => {
+            this.site_info = site_info
+        })
+        var address = document.location.search.match(/address=(.*?)[&\?]/)[1]
+        this.updateSiteblockDetails(address)
+    }
+
+    async updateSiteblockDetails(address) {
+        var address_sha256 = await sha256hex(address)
+        var blocks = await this.cmdp("siteblockList")
+        if (blocks[address] || blocks[address_sha256]) {
+            block = blocks[address]
+        } else {
+            var includes = await this.cmdp("filterIncludeList", {all_sites: true, filters: true})
+            for (let include of includes) {
+                if (include["siteblocks"][address]) {
+                    var block = include["siteblocks"][address]
+                    block["include"] = include
+                }
+                if (include["siteblocks"][address_sha256]) {
+                    var block = include["siteblocks"][address_sha256]
+                    block["include"] = include
+                }
+            }
+        }
+
+        this.blocks = blocks
+        var reason = block["reason"]
+        if (!reason) reason = "Unknown reason"
+        var date = new Date(block["date_added"] * 1000)
+        document.getElementById("reason").innerText = reason
+        document.getElementById("added").innerText = "at " + date.toLocaleDateString() + " " + date.toLocaleTimeString()
+        if (block["include"]) {
+            document.getElementById("added").innerText += " from a shared blocklist"
+            document.getElementById("visit").innerText = "Ignore blocking and visit the site"
+        }
+        document.getElementById("details").style.transform = "scale(1) rotateX(0deg)"
+        document.getElementById("visit").style.transform = "translateY(0)"
+        document.getElementById("visit").style.opacity = "1"
+        document.getElementById("visit").onclick = () => {
+            if (block["include"])
+                this.cmd("siteAdd", address, () => { this.cmd("wrapperReload") })
+            else
+                this.cmd("siteblockRemove", address, () => { this.cmd("wrapperReload") })
+        }
+    }
+}
+page = new Page()
+</script>
+</body>
+</html>

plugins/ContentFilter/media/js/ZeroFrame.js

@@ -0,0 +1,119 @@
+// Version 1.0.0 - Initial release
+// Version 1.1.0 (2017-08-02) - Added cmdp function that returns promise instead of using callback
+// Version 1.2.0 (2017-08-02) - Added Ajax monkey patch to emulate XMLHttpRequest over ZeroFrame API
+
+const CMD_INNER_READY = 'innerReady'
+const CMD_RESPONSE = 'response'
+const CMD_WRAPPER_READY = 'wrapperReady'
+const CMD_PING = 'ping'
+const CMD_PONG = 'pong'
+const CMD_WRAPPER_OPENED_WEBSOCKET = 'wrapperOpenedWebsocket'
+const CMD_WRAPPER_CLOSE_WEBSOCKET = 'wrapperClosedWebsocket'
+
+class ZeroFrame {
+    constructor(url) {
+        this.url = url
+        this.waiting_cb = {}
+        this.wrapper_nonce = document.location.href.replace(/.*wrapper_nonce=([A-Za-z0-9]+).*/, "$1")
+        this.connect()
+        this.next_message_id = 1
+        this.init()
+    }
+
+    init() {
+        return this
+    }
+
+    connect() {
+        this.target = window.parent
+        window.addEventListener('message', e => this.onMessage(e), false)
+        this.cmd(CMD_INNER_READY)
+    }
+
+    onMessage(e) {
+        let message = e.data
+        let cmd = message.cmd
+        if (cmd === CMD_RESPONSE) {
+            if (this.waiting_cb[message.to] !== undefined) {
+                this.waiting_cb[message.to](message.result)
+            }
+            else {
+                this.log("Websocket callback not found:", message)
+            }
+        } else if (cmd === CMD_WRAPPER_READY) {
+            this.cmd(CMD_INNER_READY)
+        } else if (cmd === CMD_PING) {
+            this.response(message.id, CMD_PONG)
+        } else if (cmd === CMD_WRAPPER_OPENED_WEBSOCKET) {
+            this.onOpenWebsocket()
+        } else if (cmd === CMD_WRAPPER_CLOSE_WEBSOCKET) {
+            this.onCloseWebsocket()
+        } else {
+            this.onRequest(cmd, message)
+        }
+    }
+
+    onRequest(cmd, message) {
+        this.log("Unknown request", message)
+    }
+
+    response(to, result) {
+        this.send({
+            cmd: CMD_RESPONSE,
+            to: to,
+            result: result
+        })
+    }
+
+    cmd(cmd, params={}, cb=null) {
+        this.send({
+            cmd: cmd,
+            params: params
+        }, cb)
+    }
+
+    cmdp(cmd, params={}) {
+        return new Promise((resolve, reject) => {
+            this.cmd(cmd, params, (res) => {
+                if (res && res.error) {
+                    reject(res.error)
+                } else {
+                    resolve(res)
+                }
+            })
+        })
+    }
+
+    send(message, cb=null) {
+        message.wrapper_nonce = this.wrapper_nonce
+        message.id = this.next_message_id
+        this.next_message_id++
+        this.target.postMessage(message, '*')
+        if (cb) {
+            this.waiting_cb[message.id] = cb
+        }
+    }
+
+    log(...args) {
+        console.log.apply(console, ['[ZeroFrame]'].concat(args))
+    }
+
+    onOpenWebsocket() {
+        this.log('Websocket open')
+    }
+
+    onCloseWebsocket() {
+        this.log('Websocket close')
+    }
+
+    monkeyPatchAjax() {
+        var page = this
+        XMLHttpRequest.prototype.realOpen = XMLHttpRequest.prototype.open
+        this.cmd("wrapperGetAjaxKey", [], (res) => { this.ajax_key = res })
+        var newOpen = function (method, url, async) {
+            url += "?ajax_key=" + page.ajax_key
+            return this.realOpen(method, url, async)
+        }
+        XMLHttpRequest.prototype.open = newOpen
+    }
+}

plugins/Cors/CorsPlugin.py

@@ -0,0 +1,104 @@
+import re
+import html
+import copy
+
+from Plugin import PluginManager
+from Translate import Translate
+if "_" not in locals():
+    _ = Translate("plugins/Cors/languages/")
+
+
+def getCorsPath(site, inner_path):
+    match = re.match("^cors-([A-Za-z0-9]{26,35})/(.*)", inner_path)
+    if not match:
+        raise Exception("Invalid cors path: %s" % inner_path)
+    cors_address = match.group(1)
+    cors_inner_path = match.group(2)
+
+    if not "Cors:%s" % cors_address in site.settings["permissions"]:
+        raise Exception("This site has no permission to access site %s" % cors_address)
+
+    return cors_address, cors_inner_path
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def hasSitePermission(self, address, cmd=None):
+        if super(UiWebsocketPlugin, self).hasSitePermission(address, cmd=cmd):
+            return True
+
+        if not "Cors:%s" % address in self.site.settings["permissions"] or cmd not in ["fileGet", "fileList", "dirList", "fileRules", "optionalFileInfo", "fileQuery", "dbQuery", "userGetSettings", "siteInfo"]:
+            return False
+        else:
+            return True
+
+    # Add cors support for file commands
+    def corsFuncWrapper(self, func_name, to, inner_path, *args, **kwargs):
+        if inner_path.startswith("cors-"):
+            cors_address, cors_inner_path = getCorsPath(self.site, inner_path)
+
+            req_self = copy.copy(self)
+            req_self.site = self.server.sites.get(cors_address)  # Change the site to the merged one
+            if not req_self.site:
+                return {"error": "No site found"}
+
+            func = getattr(super(UiWebsocketPlugin, req_self), func_name)
+            back = func(to, cors_inner_path, *args, **kwargs)
+            return back
+        else:
+            func = getattr(super(UiWebsocketPlugin, self), func_name)
+            return func(to, inner_path, *args, **kwargs)
+
+    def actionFileGet(self, to, inner_path, *args, **kwargs):
+        return self.corsFuncWrapper("actionFileGet", to, inner_path, *args, **kwargs)
+
+    def actionFileList(self, to, inner_path, *args, **kwargs):
+        return self.corsFuncWrapper("actionFileList", to, inner_path, *args, **kwargs)
+
+    def actionDirList(self, to, inner_path, *args, **kwargs):
+        return self.corsFuncWrapper("actionDirList", to, inner_path, *args, **kwargs)
+
+    def actionFileRules(self, to, inner_path, *args, **kwargs):
+        return self.corsFuncWrapper("actionFileRules", to, inner_path, *args, **kwargs)
+
+    def actionOptionalFileInfo(self, to, inner_path, *args, **kwargs):
+        return self.corsFuncWrapper("actionOptionalFileInfo", to, inner_path, *args, **kwargs)
+
+    def actionCorsPermission(self, to, address):
+        site = self.server.sites.get(address)
+        if site:
+            site_name = site.content_manager.contents.get("content.json", {}).get("title")
+            button_title = _["Grant"]
+        else:
+            site_name = address
+            button_title = _["Grant & Add"]
+
+        if site and "Cors:" + address in self.permissions:
+            return "ignored"
+
+        self.cmd(
+            "confirm",
+            [_["This site requests <b>read</b> permission to: <b>%s</b>"] % html.escape(site_name), button_title],
+            lambda res: self.cbCorsPermission(to, address)
+        )
+
+    def cbCorsPermission(self, to, address):
+        self.actionPermissionAdd(to, "Cors:" + address)
+        site = self.server.sites.get(address)
+        if not site:
+            self.server.site_manager.need(address)
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    # Allow to load cross origin files using /cors-address/file.jpg
+    def parsePath(self, path):
+        path_parts = super(UiRequestPlugin, self).parsePath(path)
+        if "cors-" not in path:  # Optimization
+            return path_parts
+        site = self.server.sites[path_parts["address"]]
+        try:
+            path_parts["address"], path_parts["inner_path"] = getCorsPath(site, path_parts["inner_path"])
+        except:
+            return None
+        return path_parts

plugins/Cors/__init__.py

@@ -0,0 +1 @@
+from . import CorsPlugin

plugins/CryptMessage/CryptMessage.py

@@ -0,0 +1,59 @@
+import hashlib
+import base64
+
+import lib.pybitcointools as btctools
+
+ecc_cache = {}
+
+
+def eciesEncrypt(data, pubkey, ephemcurve=None, ciphername='aes-256-cbc'):
+    import pyelliptic
+    pubkey_openssl = toOpensslPublickey(base64.b64decode(pubkey))
+    curve, pubkey_x, pubkey_y, i = pyelliptic.ECC._decode_pubkey(pubkey_openssl)
+    if ephemcurve is None:
+        ephemcurve = curve
+    ephem = pyelliptic.ECC(curve=ephemcurve)
+    key = hashlib.sha512(ephem.raw_get_ecdh_key(pubkey_x, pubkey_y)).digest()
+    key_e, key_m = key[:32], key[32:]
+    pubkey = ephem.get_pubkey()
+    iv = pyelliptic.OpenSSL.rand(pyelliptic.OpenSSL.get_cipher(ciphername).get_blocksize())
+    ctx = pyelliptic.Cipher(key_e, iv, 1, ciphername)
+    ciphertext = iv + pubkey + ctx.ciphering(data)
+    mac = pyelliptic.hmac_sha256(key_m, ciphertext)
+    return key_e, ciphertext + mac
+
+def eciesDecrypt(encrypted_data, privatekey):
+    ecc_key = getEcc(privatekey)
+    return ecc_key.decrypt(base64.b64decode(encrypted_data))
+
+def split(encrypted):
+    iv = encrypted[0:16]
+    ciphertext = encrypted[16 + 70:-32]
+
+    return iv, ciphertext
+
+
+def getEcc(privatekey=None):
+    import pyelliptic
+    global ecc_cache
+    if privatekey not in ecc_cache:
+        if privatekey:
+            publickey_bin = btctools.encode_pubkey(btctools.privtopub(privatekey), "bin")
+            publickey_openssl = toOpensslPublickey(publickey_bin)
+            privatekey_openssl = toOpensslPrivatekey(privatekey)
+            ecc_cache[privatekey] = pyelliptic.ECC(curve='secp256k1', privkey=privatekey_openssl, pubkey=publickey_openssl)
+        else:
+            ecc_cache[None] = pyelliptic.ECC()
+    return ecc_cache[privatekey]
+
+
+def toOpensslPrivatekey(privatekey):
+    privatekey_bin = btctools.encode_privkey(privatekey, "bin")
+    return b'\x02\xca\x00\x20' + privatekey_bin
+
+
+def toOpensslPublickey(publickey):
+    publickey_bin = btctools.encode_pubkey(publickey, "bin")
+    publickey_bin = publickey_bin[1:]
+    publickey_openssl = b'\x02\xca\x00 ' + publickey_bin[:32] + b'\x00 ' + publickey_bin[32:]
+    return publickey_openssl

plugins/CryptMessage/CryptMessagePlugin.py

@@ -0,0 +1,168 @@
+import base64
+import os
+
+from Plugin import PluginManager
+from Crypt import CryptBitcoin, CryptHash
+import lib.pybitcointools as btctools
+
+from . import CryptMessage
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def eciesDecrypt(self, encrypted, privatekey):
+        back = CryptMessage.getEcc(privatekey).decrypt(encrypted)
+        return back.decode("utf8")
+
+    # - Actions -
+
+    # Returns user's public key unique to site
+    # Return: Public key
+    def actionUserPublickey(self, to, index=0):
+        publickey = self.user.getEncryptPublickey(self.site.address, index)
+        self.response(to, publickey)
+
+    # Encrypt a text using the publickey or user's sites unique publickey
+    # Return: Encrypted text using base64 encoding
+    def actionEciesEncrypt(self, to, text, publickey=0, return_aes_key=False):
+        if type(publickey) is int:  # Encrypt using user's publickey
+            publickey = self.user.getEncryptPublickey(self.site.address, publickey)
+        aes_key, encrypted = CryptMessage.eciesEncrypt(text.encode("utf8"), publickey)
+        if return_aes_key:
+            self.response(to, [base64.b64encode(encrypted).decode("utf8"), base64.b64encode(aes_key).decode("utf8")])
+        else:
+            self.response(to, base64.b64encode(encrypted).decode("utf8"))
+
+    # Decrypt a text using privatekey or the user's site unique private key
+    # Return: Decrypted text or list of decrypted texts
+    def actionEciesDecrypt(self, to, param, privatekey=0):
+        if type(privatekey) is int:  # Decrypt using user's privatekey
+            privatekey = self.user.getEncryptPrivatekey(self.site.address, privatekey)
+
+        if type(param) == list:
+            encrypted_texts = param
+        else:
+            encrypted_texts = [param]
+
+        texts = []  # Decoded texts
+        for encrypted_text in encrypted_texts:
+            try:
+                text = CryptMessage.eciesDecrypt(encrypted_text, privatekey).decode("utf8")
+                texts.append(text)
+            except Exception as err:
+                texts.append(None)
+
+        if type(param) == list:
+            self.response(to, texts)
+        else:
+            self.response(to, texts[0])
+
+    # Encrypt a text using AES
+    # Return: Iv, AES key, Encrypted text
+    def actionAesEncrypt(self, to, text, key=None, iv=None):
+        import pyelliptic
+
+        if key:
+            key = base64.b64decode(key)
+        else:
+            key = os.urandom(32)
+
+        if iv:  # Generate new AES key if not definied
+            iv = base64.b64decode(iv)
+        else:
+            iv = pyelliptic.Cipher.gen_IV('aes-256-cbc')
+
+        if text:
+            encrypted = pyelliptic.Cipher(key, iv, 1, ciphername='aes-256-cbc').ciphering(text.encode("utf8"))
+        else:
+            encrypted = b""
+
+        res = [base64.b64encode(item).decode("utf8") for item in [key, iv, encrypted]]
+        self.response(to, res)
+
+    # Decrypt a text using AES
+    # Return: Decrypted text
+    def actionAesDecrypt(self, to, *args):
+        import pyelliptic
+
+        if len(args) == 3:  # Single decrypt
+            encrypted_texts = [(args[0], args[1])]
+            keys = [args[2]]
+        else:  # Batch decrypt
+            encrypted_texts, keys = args
+
+        texts = []  # Decoded texts
+        for iv, encrypted_text in encrypted_texts:
+            encrypted_text = base64.b64decode(encrypted_text)
+            iv = base64.b64decode(iv)
+            text = None
+            for key in keys:
+                ctx = pyelliptic.Cipher(base64.b64decode(key), iv, 0, ciphername='aes-256-cbc')
+                try:
+                    decrypted = ctx.ciphering(encrypted_text)
+                    if decrypted and decrypted.decode("utf8"):  # Valid text decoded
+                        text = decrypted.decode("utf8")
+                except Exception as err:
+                    pass
+            texts.append(text)
+
+        if len(args) == 3:
+            self.response(to, texts[0])
+        else:
+            self.response(to, texts)
+
+    # Sign data using ECDSA
+    # Return: Signature
+    def actionEcdsaSign(self, to, data, privatekey=None):
+        if privatekey is None:  # Sign using user's privatekey
+            privatekey = self.user.getAuthPrivatekey(self.site.address)
+
+        self.response(to, CryptBitcoin.sign(data, privatekey))
+
+    # Verify data using ECDSA (address is either a address or array of addresses)
+    # Return: bool
+    def actionEcdsaVerify(self, to, data, address, signature):
+        self.response(to, CryptBitcoin.verify(data, address, signature))
+
+    # Gets the publickey of a given privatekey
+    def actionEccPrivToPub(self, to, privatekey):
+        self.response(to, btctools.privtopub(privatekey))
+
+    # Gets the address of a given publickey
+    def actionEccPubToAddr(self, to, publickey):
+        address = btctools.pubtoaddr(btctools.decode_pubkey(publickey))
+        self.response(to, address)
+
+
+@PluginManager.registerTo("User")
+class UserPlugin(object):
+    def getEncryptPrivatekey(self, address, param_index=0):
+        assert param_index >= 0 and param_index <= 1000
+        site_data = self.getSiteData(address)
+
+        if site_data.get("cert"):  # Different privatekey for different cert provider
+            index = param_index + self.getAddressAuthIndex(site_data["cert"])
+        else:
+            index = param_index
+
+        if "encrypt_privatekey_%s" % index not in site_data:
+            address_index = self.getAddressAuthIndex(address)
+            crypt_index = address_index + 1000 + index
+            site_data["encrypt_privatekey_%s" % index] = CryptBitcoin.hdPrivatekey(self.master_seed, crypt_index)
+            self.log.debug("New encrypt privatekey generated for %s:%s" % (address, index))
+        return site_data["encrypt_privatekey_%s" % index]
+
+    def getEncryptPublickey(self, address, param_index=0):
+        assert param_index >= 0 and param_index <= 1000
+        site_data = self.getSiteData(address)
+
+        if site_data.get("cert"):  # Different privatekey for different cert provider
+            index = param_index + self.getAddressAuthIndex(site_data["cert"])
+        else:
+            index = param_index
+
+        if "encrypt_publickey_%s" % index not in site_data:
+            privatekey = self.getEncryptPrivatekey(address, param_index)
+            publickey = btctools.encode_pubkey(btctools.privtopub(privatekey), "bin_compressed")
+            site_data["encrypt_publickey_%s" % index] = base64.b64encode(publickey).decode("utf8")
+        return site_data["encrypt_publickey_%s" % index]

plugins/CryptMessage/Test/TestCrypt.py

@@ -0,0 +1,125 @@
+import pytest
+import base64
+from CryptMessage import CryptMessage
+
+
+@pytest.mark.usefixtures("resetSettings")
+class TestCrypt:
+    publickey = "A3HatibU4S6eZfIQhVs2u7GLN5G9wXa9WwlkyYIfwYaj"
+    privatekey = "5JBiKFYBm94EUdbxtnuLi6cvNcPzcKymCUHBDf2B6aq19vvG3rL"
+    utf8_text = '\xc1rv\xedzt\xfbr\xf5t\xfck\xf6rf\xfar\xf3g\xe9'
+    ecies_encrypted_text = "R5J1RFIDOzE5bnWopvccmALKACCk/CRcd/KSE9OgExJKASyMbZ57JVSUenL2TpABMmcT+wAgr2UrOqClxpOWvIUwvwwupXnMbRTzthhIJJrTRW3sCJVaYlGEMn9DAcvbflgEkQX/MVVdLV3tWKySs1Vk8sJC/y+4pGYCrZz7vwDNEEERaqU="
+
+    @pytest.mark.parametrize("text", [b"hello", '\xc1rv\xedzt\xfbr\xf5t\xfck\xf6rf\xfar\xf3g\xe9'.encode("utf8")])
+    @pytest.mark.parametrize("text_repeat", [1, 10, 128, 1024])
+    def testEncryptEcies(self, text, text_repeat):
+        text_repeated = text * text_repeat
+        aes_key, encrypted = CryptMessage.eciesEncrypt(text_repeated, self.publickey)
+        assert len(aes_key) == 32
+        # assert len(encrypted) == 134 + int(len(text) / 16) * 16  # Not always true
+
+        ecc = CryptMessage.getEcc(self.privatekey)
+        assert ecc.decrypt(encrypted) == text_repeated
+
+    def testDecryptEcies(self, user):
+        encrypted = base64.b64decode(self.ecies_encrypted_text)
+        ecc = CryptMessage.getEcc(self.privatekey)
+        assert ecc.decrypt(encrypted) == b"hello"
+
+    def testPublickey(self, ui_websocket):
+        pub = ui_websocket.testAction("UserPublickey", 0)
+        assert len(pub) == 44  # Compressed, b64 encoded publickey
+
+        # Different pubkey for specificed index
+        assert ui_websocket.testAction("UserPublickey", 1) != ui_websocket.testAction("UserPublickey", 0)
+
+        # Same publickey for same index
+        assert ui_websocket.testAction("UserPublickey", 2) == ui_websocket.testAction("UserPublickey", 2)
+
+        # Different publickey for different cert
+        site_data = ui_websocket.user.getSiteData(ui_websocket.site.address)
+        site_data["cert"] = None
+        pub1 = ui_websocket.testAction("UserPublickey", 0)
+
+        site_data = ui_websocket.user.getSiteData(ui_websocket.site.address)
+        site_data["cert"] = "zeroid.bit"
+        pub2 = ui_websocket.testAction("UserPublickey", 0)
+        assert pub1 != pub2
+
+    def testEcies(self, ui_websocket):
+        ui_websocket.actionUserPublickey(0, 0)
+        pub = ui_websocket.ws.getResult()
+
+        ui_websocket.actionEciesEncrypt(0, "hello", pub)
+        encrypted = ui_websocket.ws.getResult()
+        assert len(encrypted) == 180
+
+        # Don't allow decrypt using other privatekey index
+        ui_websocket.actionEciesDecrypt(0, encrypted, 123)
+        decrypted = ui_websocket.ws.getResult()
+        assert decrypted != "hello"
+
+        # Decrypt using correct privatekey
+        ui_websocket.actionEciesDecrypt(0, encrypted)
+        decrypted = ui_websocket.ws.getResult()
+        assert decrypted == "hello"
+
+        # Decrypt batch
+        ui_websocket.actionEciesDecrypt(0, [encrypted, "baad", encrypted])
+        decrypted = ui_websocket.ws.getResult()
+        assert decrypted == ["hello", None, "hello"]
+
+    def testEciesUtf8(self, ui_websocket):
+        # Utf8 test
+        ui_websocket.actionEciesEncrypt(0, self.utf8_text)
+        encrypted = ui_websocket.ws.getResult()
+
+        ui_websocket.actionEciesDecrypt(0, encrypted)
+        assert ui_websocket.ws.getResult() == self.utf8_text
+
+    def testEciesAes(self, ui_websocket):
+        ui_websocket.actionEciesEncrypt(0, "hello", return_aes_key=True)
+        ecies_encrypted, aes_key = ui_websocket.ws.getResult()
+
+        # Decrypt using Ecies
+        ui_websocket.actionEciesDecrypt(0, ecies_encrypted)
+        assert ui_websocket.ws.getResult() == "hello"
+
+        # Decrypt using AES
+        aes_iv, aes_encrypted = CryptMessage.split(base64.b64decode(ecies_encrypted))
+
+        ui_websocket.actionAesDecrypt(0, base64.b64encode(aes_iv), base64.b64encode(aes_encrypted), aes_key)
+        assert ui_websocket.ws.getResult() == "hello"
+
+    def testAes(self, ui_websocket):
+        ui_websocket.actionAesEncrypt(0, "hello")
+        key, iv, encrypted = ui_websocket.ws.getResult()
+
+        assert len(key) == 44
+        assert len(iv) == 24
+        assert len(encrypted) == 24
+
+        # Single decrypt
+        ui_websocket.actionAesDecrypt(0, iv, encrypted, key)
+        assert ui_websocket.ws.getResult() == "hello"
+
+        # Batch decrypt
+        ui_websocket.actionAesEncrypt(0, "hello")
+        key2, iv2, encrypted2 = ui_websocket.ws.getResult()
+
+        assert [key, iv, encrypted] != [key2, iv2, encrypted2]
+
+        # 2 correct key
+        ui_websocket.actionAesDecrypt(0, [[iv, encrypted], [iv, encrypted], [iv, "baad"], [iv2, encrypted2]], [key])
+        assert ui_websocket.ws.getResult() == ["hello", "hello", None, None]
+
+        # 3 key
+        ui_websocket.actionAesDecrypt(0, [[iv, encrypted], [iv, encrypted], [iv, "baad"], [iv2, encrypted2]], [key, key2])
+        assert ui_websocket.ws.getResult() == ["hello", "hello", None, "hello"]
+
+    def testAesUtf8(self, ui_websocket):
+        ui_websocket.actionAesEncrypt(0, self.utf8_text)
+        key, iv, encrypted = ui_websocket.ws.getResult()
+
+        ui_websocket.actionAesDecrypt(0, iv, encrypted, key)
+        assert ui_websocket.ws.getResult() == self.utf8_text

plugins/CryptMessage/Test/conftest.py

@@ -0,0 +1 @@
+from src.Test.conftest import *

plugins/CryptMessage/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/CryptMessage/__init__.py

@@ -0,0 +1 @@
+from . import CryptMessagePlugin

plugins/FilePack/FilePackPlugin.py

@@ -0,0 +1,196 @@
+import os
+import re
+
+import gevent
+
+from Plugin import PluginManager
+from Config import config
+from Debug import Debug
+
+# Keep archive open for faster reponse times for large sites
+archive_cache = {}
+
+
+def closeArchive(archive_path):
+    if archive_path in archive_cache:
+        del archive_cache[archive_path]
+
+
+def openArchive(archive_path, file_obj=None):
+    if archive_path not in archive_cache:
+        if archive_path.endswith("tar.gz"):
+            import tarfile
+            archive_cache[archive_path] = tarfile.open(archive_path, fileobj=file_obj, mode="r:gz")
+        elif archive_path.endswith("tar.bz2"):
+            import tarfile
+            archive_cache[archive_path] = tarfile.open(archive_path, fileobj=file_obj, mode="r:bz2")
+        else:
+            import zipfile
+            archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path)
+        gevent.spawn_later(5, lambda: closeArchive(archive_path))  # Close after 5 sec
+
+    archive = archive_cache[archive_path]
+    return archive
+
+
+def openArchiveFile(archive_path, path_within, file_obj=None):
+    archive = openArchive(archive_path, file_obj=file_obj)
+    if archive_path.endswith(".zip"):
+        return archive.open(path_within)
+    else:
+        return archive.extractfile(path_within)
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    def actionSiteMedia(self, path, **kwargs):
+        if ".zip/" in path or ".tar.gz/" in path:
+            file_obj = None
+            path_parts = self.parsePath(path)
+            file_path = "%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"])
+            match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path)
+            archive_path, path_within = match.groups()
+            if archive_path not in archive_cache:
+                site = self.server.site_manager.get(path_parts["address"])
+                if not site:
+                    return self.actionSiteAddPrompt(path)
+                archive_inner_path = site.storage.getInnerPath(archive_path)
+                if not os.path.isfile(archive_path):
+                    # Wait until file downloads
+                    result = site.needFile(archive_inner_path, priority=10)
+                    # Send virutal file path download finished event to remove loading screen
+                    site.updateWebsocket(file_done=archive_inner_path)
+                    if not result:
+                        return self.error404(archive_inner_path)
+                file_obj = site.storage.openBigfile(archive_inner_path)
+                if file_obj == False:
+                    file_obj = None
+
+            header_allow_ajax = False
+            if self.get.get("ajax_key"):
+                requester_site = self.server.site_manager.get(path_parts["request_address"])
+                if self.get["ajax_key"] == requester_site.settings["ajax_key"]:
+                    header_allow_ajax = True
+                else:
+                    return self.error403("Invalid ajax_key")
+
+            try:
+                file = openArchiveFile(archive_path, path_within, file_obj=file_obj)
+                content_type = self.getContentType(file_path)
+                self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax)
+                return self.streamFile(file)
+            except Exception as err:
+                self.log.debug("Error opening archive file: %s" % Debug.formatException(err))
+                return self.error404(path)
+
+        return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)
+
+    def streamFile(self, file):
+        for i in range(100):  # Read max 6MB
+            try:
+                block = file.read(60 * 1024)
+                if block:
+                    yield block
+                else:
+                    raise StopIteration
+            except StopIteration:
+                file.close()
+                break
+
+
+@PluginManager.registerTo("SiteStorage")
+class SiteStoragePlugin(object):
+    def isFile(self, inner_path):
+        if ".zip/" in inner_path or ".tar.gz/" in inner_path:
+            match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", inner_path)
+            archive_inner_path, path_within = match.groups()
+            return super(SiteStoragePlugin, self).isFile(archive_inner_path)
+        else:
+            return super(SiteStoragePlugin, self).isFile(inner_path)
+
+    def openArchive(self, inner_path):
+        archive_path = self.getPath(inner_path)
+        file_obj = None
+        if archive_path not in archive_cache:
+            if not os.path.isfile(archive_path):
+                result = self.site.needFile(inner_path, priority=10)
+                self.site.updateWebsocket(file_done=inner_path)
+                if not result:
+                    raise Exception("Unable to download file")
+            file_obj = self.site.storage.openBigfile(inner_path)
+            if file_obj == False:
+                file_obj = None
+
+        try:
+            archive = openArchive(archive_path, file_obj=file_obj)
+        except Exception as err:
+            raise Exception("Unable to download file: %s" % Debug.formatException(err))
+
+        return archive
+
+    def walk(self, inner_path, *args, **kwags):
+        if ".zip" in inner_path or ".tar.gz" in inner_path:
+            match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
+            archive_inner_path, path_within = match.groups()
+            archive = self.openArchive(archive_inner_path)
+            path_within = path_within.lstrip("/")
+
+            if archive_inner_path.endswith(".zip"):
+                namelist = [name for name in archive.namelist() if not name.endswith("/")]
+            else:
+                namelist = [item.name for item in archive.getmembers() if not item.isdir()]
+
+            namelist_relative = []
+            for name in namelist:
+                if not name.startswith(path_within):
+                    continue
+                name_relative = name.replace(path_within, "", 1).rstrip("/")
+                namelist_relative.append(name_relative)
+
+            return namelist_relative
+
+        else:
+            return super(SiteStoragePlugin, self).walk(inner_path, *args, **kwags)
+
+    def list(self, inner_path, *args, **kwags):
+        if ".zip" in inner_path or ".tar.gz" in inner_path:
+            match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
+            archive_inner_path, path_within = match.groups()
+            archive = self.openArchive(archive_inner_path)
+            path_within = path_within.lstrip("/")
+
+            if archive_inner_path.endswith(".zip"):
+                namelist = [name for name in archive.namelist()]
+            else:
+                namelist = [item.name for item in archive.getmembers()]
+
+            namelist_relative = []
+            for name in namelist:
+                if not name.startswith(path_within):
+                    continue
+                name_relative = name.replace(path_within, "", 1).rstrip("/")
+
+                if "/" in name_relative:  # File is in sub-directory
+                    continue
+
+                namelist_relative.append(name_relative)
+            return namelist_relative
+
+        else:
+            return super(SiteStoragePlugin, self).list(inner_path, *args, **kwags)
+
+    def read(self, inner_path, mode="rb"):
+        if ".zip/" in inner_path or ".tar.gz/" in inner_path:
+            match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
+            archive_inner_path, path_within = match.groups()
+            archive = self.openArchive(archive_inner_path)
+            path_within = path_within.lstrip("/")
+
+            if archive_inner_path.endswith(".zip"):
+                return archive.open(path_within).read()
+            else:
+                return archive.extractfile(path_within).read()
+
+        else:
+            return super(SiteStoragePlugin, self).read(inner_path, mode)
+

plugins/FilePack/__init__.py

@@ -0,0 +1 @@
+from . import FilePackPlugin

plugins/MergerSite/MergerSitePlugin.py

@@ -0,0 +1,384 @@
+import re
+import time
+import copy
+
+from Plugin import PluginManager
+from Translate import Translate
+from util import RateLimit
+from util import helper
+from Debug import Debug
+try:
+    import OptionalManager.UiWebsocketPlugin  # To make optioanlFileInfo merger sites compatible
+except Exception:
+    pass
+
+if "merger_db" not in locals().keys():  # To keep merger_sites between module reloads
+    merger_db = {}  # Sites that allowed to list other sites {address: [type1, type2...]}
+    merged_db = {}  # Sites that allowed to be merged to other sites {address: type, ...}
+    merged_to_merger = {}  # {address: [site1, site2, ...]} cache
+    site_manager = None  # Site manager for merger sites
+
+if "_" not in locals():
+    _ = Translate("plugins/MergerSite/languages/")
+
+
+# Check if the site has permission to this merger site
+def checkMergerPath(address, inner_path):
+    merged_match = re.match("^merged-(.*?)/([A-Za-z0-9]{26,35})/", inner_path)
+    if merged_match:
+        merger_type = merged_match.group(1)
+        # Check if merged site is allowed to include other sites
+        if merger_type in merger_db.get(address, []):
+            # Check if included site allows to include
+            merged_address = merged_match.group(2)
+            if merged_db.get(merged_address) == merger_type:
+                inner_path = re.sub("^merged-(.*?)/([A-Za-z0-9]{26,35})/", "", inner_path)
+                return merged_address, inner_path
+            else:
+                raise Exception(
+                    "Merger site (%s) does not have permission for merged site: %s (%s)" %
+                    (merger_type, merged_address, merged_db.get(merged_address))
+                )
+        else:
+            raise Exception("No merger (%s) permission to load: <br>%s (%s not in %s)" % (
+                address, inner_path, merger_type, merger_db.get(address, []))
+            )
+    else:
+        raise Exception("Invalid merger path: %s" % inner_path)
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    # Download new site
+    def actionMergerSiteAdd(self, to, addresses):
+        if type(addresses) != list:
+            # Single site add
+            addresses = [addresses]
+        # Check if the site has merger permission
+        merger_types = merger_db.get(self.site.address)
+        if not merger_types:
+            return self.response(to, {"error": "Not a merger site"})
+
+        if RateLimit.isAllowed(self.site.address + "-MergerSiteAdd", 10) and len(addresses) == 1:
+            # Without confirmation if only one site address and not called in last 10 sec
+            self.cbMergerSiteAdd(to, addresses)
+        else:
+            self.cmd(
+                "confirm",
+                [_["Add <b>%s</b> new site?"] % len(addresses), "Add"],
+                lambda res: self.cbMergerSiteAdd(to, addresses)
+            )
+        self.response(to, "ok")
+
+    # Callback of adding new site confirmation
+    def cbMergerSiteAdd(self, to, addresses):
+        added = 0
+        for address in addresses:
+            added += 1
+            site_manager.need(address)
+        if added:
+            self.cmd("notification", ["done", _["Added <b>%s</b> new site"] % added, 5000])
+        RateLimit.called(self.site.address + "-MergerSiteAdd")
+        site_manager.updateMergerSites()
+
+    # Delete a merged site
+    def actionMergerSiteDelete(self, to, address):
+        site = self.server.sites.get(address)
+        if not site:
+            return self.response(to, {"error": "No site found: %s" % address})
+
+        merger_types = merger_db.get(self.site.address)
+        if not merger_types:
+            return self.response(to, {"error": "Not a merger site"})
+        if merged_db.get(address) not in merger_types:
+            return self.response(to, {"error": "Merged type (%s) not in %s" % (merged_db.get(address), merger_types)})
+
+        self.cmd("notification", ["done", _["Site deleted: <b>%s</b>"] % address, 5000])
+        self.response(to, "ok")
+
+    # Lists merged sites
+    def actionMergerSiteList(self, to, query_site_info=False):
+        merger_types = merger_db.get(self.site.address)
+        ret = {}
+        if not merger_types:
+            return self.response(to, {"error": "Not a merger site"})
+        for address, merged_type in merged_db.items():
+            if merged_type not in merger_types:
+                continue  # Site not for us
+            if query_site_info:
+                site = self.server.sites.get(address)
+                ret[address] = self.formatSiteInfo(site, create_user=False)
+            else:
+                ret[address] = merged_type
+        self.response(to, ret)
+
+    def hasSitePermission(self, address, *args, **kwargs):
+        if super(UiWebsocketPlugin, self).hasSitePermission(address, *args, **kwargs):
+            return True
+        else:
+            if self.site.address in [merger_site.address for merger_site in merged_to_merger.get(address, [])]:
+                return True
+            else:
+                return False
+
+    # Add support merger sites for file commands
+    def mergerFuncWrapper(self, func_name, to, inner_path, *args, **kwargs):
+        if inner_path.startswith("merged-"):
+            merged_address, merged_inner_path = checkMergerPath(self.site.address, inner_path)
+
+            # Set the same cert for merged site
+            merger_cert = self.user.getSiteData(self.site.address).get("cert")
+            if merger_cert and self.user.getSiteData(merged_address).get("cert") != merger_cert:
+                self.user.setCert(merged_address, merger_cert)
+
+            req_self = copy.copy(self)
+            req_self.site = self.server.sites.get(merged_address)  # Change the site to the merged one
+
+            func = getattr(super(UiWebsocketPlugin, req_self), func_name)
+            return func(to, merged_inner_path, *args, **kwargs)
+        else:
+            func = getattr(super(UiWebsocketPlugin, self), func_name)
+            return func(to, inner_path, *args, **kwargs)
+
+    def actionFileList(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionFileList", to, inner_path, *args, **kwargs)
+
+    def actionDirList(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionDirList", to, inner_path, *args, **kwargs)
+
+    def actionFileGet(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionFileGet", to, inner_path, *args, **kwargs)
+
+    def actionFileWrite(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionFileWrite", to, inner_path, *args, **kwargs)
+
+    def actionFileDelete(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionFileDelete", to, inner_path, *args, **kwargs)
+
+    def actionFileRules(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionFileRules", to, inner_path, *args, **kwargs)
+
+    def actionFileNeed(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionFileNeed", to, inner_path, *args, **kwargs)
+
+    def actionOptionalFileInfo(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionOptionalFileInfo", to, inner_path, *args, **kwargs)
+
+    def actionOptionalFileDelete(self, to, inner_path, *args, **kwargs):
+        return self.mergerFuncWrapper("actionOptionalFileDelete", to, inner_path, *args, **kwargs)
+
+    def actionBigfileUploadInit(self, to, inner_path, *args, **kwargs):
+        back = self.mergerFuncWrapper("actionBigfileUploadInit", to, inner_path, *args, **kwargs)
+        if inner_path.startswith("merged-"):
+            merged_address, merged_inner_path = checkMergerPath(self.site.address, inner_path)
+            back["inner_path"] = "merged-%s/%s/%s" % (merged_db[merged_address], merged_address, back["inner_path"])
+        return back
+
+    # Add support merger sites for file commands with privatekey parameter
+    def mergerFuncWrapperWithPrivatekey(self, func_name, to, privatekey, inner_path, *args, **kwargs):
+        func = getattr(super(UiWebsocketPlugin, self), func_name)
+        if inner_path.startswith("merged-"):
+            merged_address, merged_inner_path = checkMergerPath(self.site.address, inner_path)
+            merged_site = self.server.sites.get(merged_address)
+
+            # Set the same cert for merged site
+            merger_cert = self.user.getSiteData(self.site.address).get("cert")
+            if merger_cert:
+                self.user.setCert(merged_address, merger_cert)
+
+            site_before = self.site  # Save to be able to change it back after we ran the command
+            self.site = merged_site  # Change the site to the merged one
+            try:
+                back = func(to, privatekey, merged_inner_path, *args, **kwargs)
+            finally:
+                self.site = site_before  # Change back to original site
+            return back
+        else:
+            return func(to, privatekey, inner_path, *args, **kwargs)
+
+    def actionSiteSign(self, to, privatekey=None, inner_path="content.json", *args, **kwargs):
+        return self.mergerFuncWrapperWithPrivatekey("actionSiteSign", to, privatekey, inner_path, *args, **kwargs)
+
+    def actionSitePublish(self, to, privatekey=None, inner_path="content.json", *args, **kwargs):
+        return self.mergerFuncWrapperWithPrivatekey("actionSitePublish", to, privatekey, inner_path, *args, **kwargs)
+
+    def actionPermissionAdd(self, to, permission):
+        super(UiWebsocketPlugin, self).actionPermissionAdd(to, permission)
+        if permission.startswith("Merger"):
+            self.site.storage.rebuildDb()
+
+    def actionPermissionDetails(self, to, permission):
+        if not permission.startswith("Merger"):
+            return super(UiWebsocketPlugin, self).actionPermissionDetails(to, permission)
+
+        merger_type = permission.replace("Merger:", "")
+        if not re.match("^[A-Za-z0-9-]+$", merger_type):
+            raise Exception("Invalid merger_type: %s" % merger_type)
+        merged_sites = []
+        for address, merged_type in merged_db.items():
+            if merged_type != merger_type:
+                continue
+            site = self.server.sites.get(address)
+            try:
+                merged_sites.append(site.content_manager.contents.get("content.json").get("title", address))
+            except Exception as err:
+                merged_sites.append(address)
+
+        details = _["Read and write permissions to sites with merged type of <b>%s</b> "] % merger_type
+        details += _["(%s sites)"] % len(merged_sites)
+        details += "<div style='white-space: normal; max-width: 400px'>%s</div>" % ", ".join(merged_sites)
+        self.response(to, details)
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    # Allow to load merged site files using /merged-ZeroMe/address/file.jpg
+    def parsePath(self, path):
+        path_parts = super(UiRequestPlugin, self).parsePath(path)
+        if "merged-" not in path:  # Optimization
+            return path_parts
+        path_parts["address"], path_parts["inner_path"] = checkMergerPath(path_parts["address"], path_parts["inner_path"])
+        return path_parts
+
+
+@PluginManager.registerTo("SiteStorage")
+class SiteStoragePlugin(object):
+    # Also rebuild from merged sites
+    def getDbFiles(self):
+        merger_types = merger_db.get(self.site.address)
+
+        # First return the site's own db files
+        for item in super(SiteStoragePlugin, self).getDbFiles():
+            yield item
+
+        # Not a merger site, that's all
+        if not merger_types:
+            return
+
+        merged_sites = [
+            site_manager.sites[address]
+            for address, merged_type in merged_db.items()
+            if merged_type in merger_types
+        ]
+        found = 0
+        for merged_site in merged_sites:
+            self.log.debug("Loading merged site: %s" % merged_site)
+            merged_type = merged_db[merged_site.address]
+            for content_inner_path, content in merged_site.content_manager.contents.items():
+                # content.json file itself
+                if merged_site.storage.isFile(content_inner_path):  # Missing content.json file
+                    merged_inner_path = "merged-%s/%s/%s" % (merged_type, merged_site.address, content_inner_path)
+                    yield merged_inner_path, merged_site.storage.getPath(content_inner_path)
+                else:
+                    merged_site.log.error("[MISSING] %s" % content_inner_path)
+                # Data files in content.json
+                content_inner_path_dir = helper.getDirname(content_inner_path)  # Content.json dir relative to site
+                for file_relative_path in list(content.get("files", {}).keys()) + list(content.get("files_optional", {}).keys()):
+                    if not file_relative_path.endswith(".json"):
+                        continue  # We only interesed in json files
+                    file_inner_path = content_inner_path_dir + file_relative_path  # File Relative to site dir
+                    file_inner_path = file_inner_path.strip("/")  # Strip leading /
+                    if merged_site.storage.isFile(file_inner_path):
+                        merged_inner_path = "merged-%s/%s/%s" % (merged_type, merged_site.address, file_inner_path)
+                        yield merged_inner_path, merged_site.storage.getPath(file_inner_path)
+                    else:
+                        merged_site.log.error("[MISSING] %s" % file_inner_path)
+                    found += 1
+                    if found % 100 == 0:
+                        time.sleep(0.001)  # Context switch to avoid UI block
+
+    # Also notice merger sites on a merged site file change
+    def onUpdated(self, inner_path, file=None):
+        super(SiteStoragePlugin, self).onUpdated(inner_path, file)
+
+        merged_type = merged_db.get(self.site.address)
+
+        for merger_site in merged_to_merger.get(self.site.address, []):
+            if merger_site.address == self.site.address:  # Avoid infinite loop
+                continue
+            virtual_path = "merged-%s/%s/%s" % (merged_type, self.site.address, inner_path)
+            if inner_path.endswith(".json"):
+                if file is not None:
+                    merger_site.storage.onUpdated(virtual_path, file=file)
+                else:
+                    merger_site.storage.onUpdated(virtual_path, file=self.open(inner_path))
+            else:
+                merger_site.storage.onUpdated(virtual_path)
+
+
+@PluginManager.registerTo("Site")
+class SitePlugin(object):
+    def fileDone(self, inner_path):
+        super(SitePlugin, self).fileDone(inner_path)
+
+        for merger_site in merged_to_merger.get(self.address, []):
+            if merger_site.address == self.address:
+                continue
+            for ws in merger_site.websockets:
+                ws.event("siteChanged", self, {"event": ["file_done", inner_path]})
+
+    def fileFailed(self, inner_path):
+        super(SitePlugin, self).fileFailed(inner_path)
+
+        for merger_site in merged_to_merger.get(self.address, []):
+            if merger_site.address == self.address:
+                continue
+            for ws in merger_site.websockets:
+                ws.event("siteChanged", self, {"event": ["file_failed", inner_path]})
+
+
+@PluginManager.registerTo("SiteManager")
+class SiteManagerPlugin(object):
+    # Update merger site for site types
+    def updateMergerSites(self):
+        global merger_db, merged_db, merged_to_merger, site_manager
+        s = time.time()
+        merger_db = {}
+        merged_db = {}
+        merged_to_merger = {}
+        site_manager = self
+        if not self.sites:
+            return
+        for site in self.sites.values():
+            # Update merged sites
+            try:
+                merged_type = site.content_manager.contents.get("content.json", {}).get("merged_type")
+            except Exception as err:
+                self.log.error("Error loading site %s: %s" % (site.address, Debug.formatException(err)))
+                continue
+            if merged_type:
+                merged_db[site.address] = merged_type
+
+            # Update merger sites
+            for permission in site.settings["permissions"]:
+                if not permission.startswith("Merger:"):
+                    continue
+                if merged_type:
+                    self.log.error(
+                        "Removing permission %s from %s: Merger and merged at the same time." %
+                        (permission, site.address)
+                    )
+                    site.settings["permissions"].remove(permission)
+                    continue
+                merger_type = permission.replace("Merger:", "")
+                if site.address not in merger_db:
+                    merger_db[site.address] = []
+                merger_db[site.address].append(merger_type)
+                site_manager.sites[site.address] = site
+
+            # Update merged to merger
+            if merged_type:
+                for merger_site in self.sites.values():
+                    if "Merger:" + merged_type in merger_site.settings["permissions"]:
+                        if site.address not in merged_to_merger:
+                            merged_to_merger[site.address] = []
+                        merged_to_merger[site.address].append(merger_site)
+        self.log.debug("Updated merger sites in %.3fs" % (time.time() - s))
+
+    def load(self, *args, **kwags):
+        super(SiteManagerPlugin, self).load(*args, **kwags)
+        self.updateMergerSites()
+
+    def save(self, *args, **kwags):
+        super(SiteManagerPlugin, self).save(*args, **kwags)
+        self.updateMergerSites()

plugins/MergerSite/__init__.py

@@ -0,0 +1 @@
+from . import MergerSitePlugin

plugins/MergerSite/languages/es.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "¿Agregar <b>%s</b> nuevo sitio?",
+	"Added <b>%s</b> new site": "Sitio <b>%s</b> agregado",
+	"Site deleted: <b>%s</b>": "Sitio removido: <b>%s</b>"
+}

plugins/MergerSite/languages/fr.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "Ajouter le site <b>%s</b> ?",
+	"Added <b>%s</b> new site": "Site <b>%s</b> ajouté",
+	"Site deleted: <b>%s</b>": "Site <b>%s</b> supprimé"
+}

plugins/MergerSite/languages/hu.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "Új oldal hozzáadása: <b>%s</b>?",
+	"Added <b>%s</b> new site": "Új oldal hozzáadva: <b>%s</b>",
+	"Site deleted: <b>%s</b>": "Oldal törölve: <b>%s</b>"
+}

plugins/MergerSite/languages/it.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "Aggiungere <b>%s</b> nuovo sito ?",
+	"Added <b>%s</b> new site": "Sito <b>%s</b> aggiunto",
+	"Site deleted: <b>%s</b>": "Sito <b>%s</b> eliminato"
+}

plugins/MergerSite/languages/pt-br.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "Adicionar <b>%s</b> novo site?",
+	"Added <b>%s</b> new site": "Site <b>%s</b> adicionado",
+	"Site deleted: <b>%s</b>": "Site removido: <b>%s</b>"
+}

plugins/MergerSite/languages/tr.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "<b>%s</b> sitesi eklensin mi?",
+	"Added <b>%s</b> new site": "<b>%s</b> sitesi eklendi",
+	"Site deleted: <b>%s</b>": "<b>%s</b> sitesi silindi"
+}

plugins/MergerSite/languages/zh-tw.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "添加新網站: <b>%s</b>？",
+	"Added <b>%s</b> new site": "已添加到新網站：<b>%s</b>",
+	"Site deleted: <b>%s</b>": "網站已刪除：<b>%s</b>"
+}

plugins/MergerSite/languages/zh.json

@@ -0,0 +1,5 @@
+{
+	"Add <b>%s</b> new site?": "添加新站点: <b>%s</b>？",
+	"Added <b>%s</b> new site": "已添加到新站点：<b>%s</b>",
+	"Site deleted: <b>%s</b>": "站点已删除：<b>%s</b>"
+}

plugins/Newsfeed/NewsfeedPlugin.py

@@ -0,0 +1,188 @@
+import time
+import re
+
+from Plugin import PluginManager
+from Db.DbQuery import DbQuery
+from Debug import Debug
+from util import helper
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def formatSiteInfo(self, site, create_user=True):
+        site_info = super(UiWebsocketPlugin, self).formatSiteInfo(site, create_user=create_user)
+        feed_following = self.user.sites.get(site.address, {}).get("follow", None)
+        if feed_following == None:
+            site_info["feed_follow_num"] = None
+        else:
+            site_info["feed_follow_num"] = len(feed_following)
+        return site_info
+
+    def actionFeedFollow(self, to, feeds):
+        self.user.setFeedFollow(self.site.address, feeds)
+        self.user.save()
+        self.response(to, "ok")
+
+    def actionFeedListFollow(self, to):
+        feeds = self.user.sites[self.site.address].get("follow", {})
+        self.response(to, feeds)
+
+    def actionFeedQuery(self, to, limit=10, day_limit=3):
+        if "ADMIN" not in self.site.settings["permissions"]:
+            return self.response(to, "FeedQuery not allowed")
+
+        from Site import SiteManager
+        rows = []
+        stats = []
+
+        total_s = time.time()
+        num_sites = 0
+
+        for address, site_data in list(self.user.sites.items()):
+            feeds = site_data.get("follow")
+            if not feeds:
+                continue
+            if type(feeds) is not dict:
+                self.log.debug("Invalid feed for site %s" % address)
+                continue
+            num_sites += 1
+            for name, query_set in feeds.items():
+                site = SiteManager.site_manager.get(address)
+                if not site or not site.storage.has_db:
+                    continue
+
+                s = time.time()
+                try:
+                    query_raw, params = query_set
+                    query_parts = re.split(r"UNION(?:\s+ALL|)", query_raw)
+                    for i, query_part in enumerate(query_parts):
+                        db_query = DbQuery(query_part)
+                        if day_limit:
+                            where = " WHERE %s > strftime('%%s', 'now', '-%s day')" % (db_query.fields.get("date_added", "date_added"), day_limit)
+                            if "WHERE" in query_part:
+                                query_part = re.sub("WHERE (.*?)(?=$| GROUP BY)", where+" AND (\\1)", query_part)
+                            else:
+                                query_part += where
+                        query_parts[i] = query_part
+                    query = " UNION ".join(query_parts)
+
+                    if ":params" in query:
+                        query_params = map(helper.sqlquote, params)
+                        query = query.replace(":params", ",".join(query_params))
+
+                    res = site.storage.query(query + " ORDER BY date_added DESC LIMIT %s" % limit)
+
+                except Exception as err:  # Log error
+                    self.log.error("%s feed query %s error: %s" % (address, name, Debug.formatException(err)))
+                    stats.append({"site": site.address, "feed_name": name, "error": str(err)})
+                    continue
+
+                for row in res:
+                    row = dict(row)
+                    if not isinstance(row["date_added"], (int, float, complex)):
+                        self.log.debug("Invalid date_added from site %s: %r" % (address, row["date_added"]))
+                        continue
+                    if row["date_added"] > 1000000000000:  # Formatted as millseconds
+                        row["date_added"] = row["date_added"] / 1000
+                    if "date_added" not in row or row["date_added"] > time.time() + 120:
+                        self.log.debug("Newsfeed item from the future from from site %s" % address)
+                        continue  # Feed item is in the future, skip it
+                    row["site"] = address
+                    row["feed_name"] = name
+                    rows.append(row)
+                stats.append({"site": site.address, "feed_name": name, "taken": round(time.time() - s, 3)})
+                time.sleep(0.001)
+        return self.response(to, {"rows": rows, "stats": stats, "num": len(rows), "sites": num_sites, "taken": round(time.time() - total_s, 3)})
+
+    def parseSearch(self, search):
+        parts = re.split("(site|type):", search)
+        if len(parts) > 1:  # Found filter
+            search_text = parts[0]
+            parts = [part.strip() for part in parts]
+            filters = dict(zip(parts[1::2], parts[2::2]))
+        else:
+            search_text = search
+            filters = {}
+        return [search_text, filters]
+
+    def actionFeedSearch(self, to, search, limit=30, day_limit=30):
+        if "ADMIN" not in self.site.settings["permissions"]:
+            return self.response(to, "FeedSearch not allowed")
+
+        from Site import SiteManager
+        rows = []
+        stats = []
+        num_sites = 0
+        total_s = time.time()
+
+        search_text, filters = self.parseSearch(search)
+
+        for address, site in SiteManager.site_manager.list().items():
+            if not site.storage.has_db:
+                continue
+
+            if "site" in filters:
+                if filters["site"].lower() not in [site.address, site.content_manager.contents["content.json"].get("title").lower()]:
+                    continue
+
+            if site.storage.db:  # Database loaded
+                feeds = site.storage.db.schema.get("feeds")
+            else:
+                try:
+                    feeds = site.storage.loadJson("dbschema.json").get("feeds")
+                except:
+                    continue
+
+            if not feeds:
+                continue
+
+            num_sites += 1
+
+            for name, query in feeds.items():
+                s = time.time()
+                try:
+                    db_query = DbQuery(query)
+
+                    params = []
+                    # Filters
+                    if search_text:
+                        db_query.wheres.append("(%s LIKE ? OR %s LIKE ?)" % (db_query.fields["body"], db_query.fields["title"]))
+                        search_like = "%" + search_text.replace(" ", "%") + "%"
+                        params.append(search_like)
+                        params.append(search_like)
+                    if filters.get("type") and filters["type"] not in query:
+                        continue
+
+                    if day_limit:
+                        db_query.wheres.append(
+                            "%s > strftime('%%s', 'now', '-%s day')" % (db_query.fields.get("date_added", "date_added"), day_limit)
+                        )
+
+                    # Order
+                    db_query.parts["ORDER BY"] = "date_added DESC"
+                    db_query.parts["LIMIT"] = str(limit)
+
+                    res = site.storage.query(str(db_query), params)
+                except Exception as err:
+                    self.log.error("%s feed query %s error: %s" % (address, name, Debug.formatException(err)))
+                    stats.append({"site": site.address, "feed_name": name, "error": str(err), "query": query})
+                    continue
+                for row in res:
+                    row = dict(row)
+                    if not row["date_added"] or row["date_added"] > time.time() + 120:
+                        continue  # Feed item is in the future, skip it
+                    row["site"] = address
+                    row["feed_name"] = name
+                    rows.append(row)
+                stats.append({"site": site.address, "feed_name": name, "taken": round(time.time() - s, 3)})
+        return self.response(to, {"rows": rows, "num": len(rows), "sites": num_sites, "taken": round(time.time() - total_s, 3), "stats": stats})
+
+
+@PluginManager.registerTo("User")
+class UserPlugin(object):
+    # Set queries that user follows
+    def setFeedFollow(self, address, feeds):
+        site_data = self.getSiteData(address)
+        site_data["follow"] = feeds
+        self.save()
+        return site_data

plugins/Newsfeed/__init__.py

@@ -0,0 +1 @@
+from . import NewsfeedPlugin

plugins/OptionalManager/ContentDbPlugin.py

@@ -0,0 +1,407 @@
+import time
+import collections
+import itertools
+import re
+
+import gevent
+
+from util import helper
+from Plugin import PluginManager
+from Config import config
+from Debug import Debug
+
+if "content_db" not in locals().keys():  # To keep between module reloads
+    content_db = None
+
+
+@PluginManager.registerTo("ContentDb")
+class ContentDbPlugin(object):
+    def __init__(self, *args, **kwargs):
+        global content_db
+        content_db = self
+        self.filled = {}  # Site addresses that already filled from content.json
+        self.need_filling = False  # file_optional table just created, fill data from content.json files
+        self.time_peer_numbers_updated = 0
+        self.my_optional_files = {}  # Last 50 site_address/inner_path called by fileWrite (auto-pinning these files)
+        self.optional_files = collections.defaultdict(dict)
+        self.optional_files_loading = False
+        helper.timer(60 * 5, self.checkOptionalLimit)
+        super(ContentDbPlugin, self).__init__(*args, **kwargs)
+
+    def getSchema(self):
+        schema = super(ContentDbPlugin, self).getSchema()
+
+        # Need file_optional table
+        schema["tables"]["file_optional"] = {
+            "cols": [
+                ["file_id", "INTEGER PRIMARY KEY UNIQUE NOT NULL"],
+                ["site_id", "INTEGER REFERENCES site (site_id) ON DELETE CASCADE"],
+                ["inner_path", "TEXT"],
+                ["hash_id", "INTEGER"],
+                ["size", "INTEGER"],
+                ["peer", "INTEGER DEFAULT 0"],
+                ["uploaded", "INTEGER DEFAULT 0"],
+                ["is_downloaded", "INTEGER DEFAULT 0"],
+                ["is_pinned", "INTEGER DEFAULT 0"],
+                ["time_added", "INTEGER DEFAULT 0"],
+                ["time_downloaded", "INTEGER DEFAULT 0"],
+                ["time_accessed", "INTEGER DEFAULT 0"]
+            ],
+            "indexes": [
+                "CREATE UNIQUE INDEX file_optional_key ON file_optional (site_id, inner_path)",
+                "CREATE INDEX is_downloaded ON file_optional (is_downloaded)"
+            ],
+            "schema_changed": 11
+        }
+
+        return schema
+
+    def initSite(self, site):
+        super(ContentDbPlugin, self).initSite(site)
+        if self.need_filling:
+            self.fillTableFileOptional(site)
+        if not self.optional_files_loading:
+            gevent.spawn_later(1, self.loadFilesOptional)
+            self.optional_files_loading = True
+
+    def checkTables(self):
+        changed_tables = super(ContentDbPlugin, self).checkTables()
+        if "file_optional" in changed_tables:
+            self.need_filling = True
+        return changed_tables
+
+    # Load optional files ending
+    def loadFilesOptional(self):
+        s = time.time()
+        num = 0
+        total = 0
+        total_downloaded = 0
+        res = content_db.execute("SELECT site_id, inner_path, size, is_downloaded FROM file_optional")
+        site_sizes = collections.defaultdict(lambda: collections.defaultdict(int))
+        for row in res:
+            self.optional_files[row["site_id"]][row["inner_path"][-8:]] = 1
+            num += 1
+
+            # Update site size stats
+            site_sizes[row["site_id"]]["size_optional"] += row["size"]
+            if row["is_downloaded"]:
+                site_sizes[row["site_id"]]["optional_downloaded"] += row["size"]
+
+        # Site site size stats to sites.json settings
+        site_ids_reverse = {val: key for key, val in self.site_ids.items()}
+        for site_id, stats in site_sizes.items():
+            site_address = site_ids_reverse.get(site_id)
+            if not site_address:
+                self.log.error("Not found site_id: %s" % site_id)
+                continue
+            site = self.sites[site_address]
+            site.settings["size_optional"] = stats["size_optional"]
+            site.settings["optional_downloaded"] = stats["optional_downloaded"]
+            total += stats["size_optional"]
+            total_downloaded += stats["optional_downloaded"]
+
+        self.log.debug(
+            "Loaded %s optional files: %.2fMB, downloaded: %.2fMB in %.3fs" %
+            (num, float(total) / 1024 / 1024, float(total_downloaded) / 1024 / 1024, time.time() - s)
+        )
+
+        if self.need_filling and self.getOptionalLimitBytes() >= 0 and self.getOptionalLimitBytes() < total_downloaded:
+            limit_bytes = self.getOptionalLimitBytes()
+            limit_new = round((float(total_downloaded) / 1024 / 1024 / 1024) * 1.1, 2)  # Current limit + 10%
+            self.log.debug(
+                "First startup after update and limit is smaller than downloaded files size (%.2fGB), increasing it from %.2fGB to %.2fGB" %
+                (float(total_downloaded) / 1024 / 1024 / 1024, float(limit_bytes) / 1024 / 1024 / 1024, limit_new)
+            )
+            config.saveValue("optional_limit", limit_new)
+            config.optional_limit = str(limit_new)
+
+    # Predicts if the file is optional
+    def isOptionalFile(self, site_id, inner_path):
+        return self.optional_files[site_id].get(inner_path[-8:])
+
+    # Fill file_optional table with optional files found in sites
+    def fillTableFileOptional(self, site):
+        s = time.time()
+        site_id = self.site_ids.get(site.address)
+        if not site_id:
+            return False
+        cur = self.getCursor()
+        res = cur.execute("SELECT * FROM content WHERE size_files_optional > 0 AND site_id = %s" % site_id)
+        num = 0
+        for row in res.fetchall():
+            content = site.content_manager.contents[row["inner_path"]]
+            try:
+                num += self.setContentFilesOptional(site, row["inner_path"], content, cur=cur)
+            except Exception as err:
+                self.log.error("Error loading %s into file_optional: %s" % (row["inner_path"], err))
+        cur.close()
+
+        # Set my files to pinned
+        from User import UserManager
+        user = UserManager.user_manager.get()
+        if not user:
+            user = UserManager.user_manager.create()
+        auth_address = user.getAuthAddress(site.address)
+        self.execute(
+            "UPDATE file_optional SET is_pinned = 1 WHERE site_id = :site_id AND inner_path LIKE :inner_path",
+            {"site_id": site_id, "inner_path": "%%/%s/%%" % auth_address}
+        )
+
+        self.log.debug(
+            "Filled file_optional table for %s in %.3fs (loaded: %s, is_pinned: %s)" %
+            (site.address, time.time() - s, num, self.cur.cursor.rowcount)
+        )
+        self.filled[site.address] = True
+
+    def setContentFilesOptional(self, site, content_inner_path, content, cur=None):
+        if not cur:
+            cur = self
+
+        num = 0
+        site_id = self.site_ids[site.address]
+        content_inner_dir = helper.getDirname(content_inner_path)
+        for relative_inner_path, file in content.get("files_optional", {}).items():
+            file_inner_path = content_inner_dir + relative_inner_path
+            hash_id = int(file["sha512"][0:4], 16)
+            if hash_id in site.content_manager.hashfield:
+                is_downloaded = 1
+            else:
+                is_downloaded = 0
+            if site.address + "/" + content_inner_dir in self.my_optional_files:
+                is_pinned = 1
+            else:
+                is_pinned = 0
+            cur.insertOrUpdate("file_optional", {
+                "hash_id": hash_id,
+                "size": int(file["size"])
+            }, {
+                "site_id": site_id,
+                "inner_path": file_inner_path
+            }, oninsert={
+                "time_added": int(time.time()),
+                "time_downloaded": int(time.time()) if is_downloaded else 0,
+                "is_downloaded": is_downloaded,
+                "peer": is_downloaded,
+                "is_pinned": is_pinned
+            })
+            self.optional_files[site_id][file_inner_path[-8:]] = 1
+            num += 1
+
+        return num
+
+    def setContent(self, site, inner_path, content, size=0):
+        super(ContentDbPlugin, self).setContent(site, inner_path, content, size=size)
+        old_content = site.content_manager.contents.get(inner_path, {})
+        if (not self.need_filling or self.filled.get(site.address)) and ("files_optional" in content or "files_optional" in old_content):
+            self.setContentFilesOptional(site, inner_path, content)
+            # Check deleted files
+            if old_content:
+                old_files = old_content.get("files_optional", {}).keys()
+                new_files = content.get("files_optional", {}).keys()
+                content_inner_dir = helper.getDirname(inner_path)
+                deleted = [content_inner_dir + key for key in old_files if key not in new_files]
+                if deleted:
+                    site_id = self.site_ids[site.address]
+                    self.execute("DELETE FROM file_optional WHERE ?", {"site_id": site_id, "inner_path": deleted})
+
+    def deleteContent(self, site, inner_path):
+        content = site.content_manager.contents.get(inner_path)
+        if content and "files_optional" in content:
+            site_id = self.site_ids[site.address]
+            content_inner_dir = helper.getDirname(inner_path)
+            optional_inner_paths = [
+                content_inner_dir + relative_inner_path
+                for relative_inner_path in content.get("files_optional", {}).keys()
+            ]
+            self.execute("DELETE FROM file_optional WHERE ?", {"site_id": site_id, "inner_path": optional_inner_paths})
+        super(ContentDbPlugin, self).deleteContent(site, inner_path)
+
+    def updatePeerNumbers(self):
+        s = time.time()
+        num_file = 0
+        num_updated = 0
+        num_site = 0
+        for site in list(self.sites.values()):
+            if not site.content_manager.has_optional_files:
+                continue
+            if not site.isServing():
+                continue
+            has_updated_hashfield = next((
+                peer
+                for peer in site.peers.values()
+                if peer.has_hashfield and peer.hashfield.time_changed > self.time_peer_numbers_updated
+            ), None)
+
+            if not has_updated_hashfield and site.content_manager.hashfield.time_changed < self.time_peer_numbers_updated:
+                continue
+
+            hashfield_peers = itertools.chain.from_iterable(
+                peer.hashfield.storage
+                for peer in site.peers.values()
+                if peer.has_hashfield
+            )
+            peer_nums = collections.Counter(
+                itertools.chain(
+                    hashfield_peers,
+                    site.content_manager.hashfield
+                )
+            )
+
+            site_id = self.site_ids[site.address]
+            if not site_id:
+                continue
+
+            res = self.execute("SELECT file_id, hash_id, peer FROM file_optional WHERE ?", {"site_id": site_id})
+            updates = {}
+            for row in res:
+                peer_num = peer_nums.get(row["hash_id"], 0)
+                if peer_num != row["peer"]:
+                    updates[row["file_id"]] = peer_num
+
+            for file_id, peer_num in updates.items():
+                self.execute("UPDATE file_optional SET peer = ? WHERE file_id = ?", (peer_num, file_id))
+
+            num_updated += len(updates)
+            num_file += len(peer_nums)
+            num_site += 1
+
+        self.time_peer_numbers_updated = time.time()
+        self.log.debug("%s/%s peer number for %s site updated in %.3fs" % (num_updated, num_file, num_site, time.time() - s))
+
+    def queryDeletableFiles(self):
+        # First return the files with atleast 10 seeder and not accessed in last week
+        query = """
+            SELECT * FROM file_optional
+            WHERE peer > 10 AND %s
+            ORDER BY time_accessed < %s DESC, uploaded / size
+        """ % (self.getOptionalUsedWhere(), int(time.time() - 60 * 60 * 7))
+        limit_start = 0
+        while 1:
+            num = 0
+            res = self.execute("%s LIMIT %s, 50" % (query, limit_start))
+            for row in res:
+                yield row
+                num += 1
+            if num < 50:
+                break
+            limit_start += 50
+
+        self.log.debug("queryDeletableFiles returning less-seeded files")
+
+        # Then return files less seeder but still not accessed in last week
+        query = """
+            SELECT * FROM file_optional
+            WHERE peer <= 10 AND %s
+            ORDER BY peer DESC, time_accessed < %s DESC, uploaded / size
+        """ % (self.getOptionalUsedWhere(), int(time.time() - 60 * 60 * 7))
+        limit_start = 0
+        while 1:
+            num = 0
+            res = self.execute("%s LIMIT %s, 50" % (query, limit_start))
+            for row in res:
+                yield row
+                num += 1
+            if num < 50:
+                break
+            limit_start += 50
+
+        self.log.debug("queryDeletableFiles returning everyting")
+
+        # At the end return all files
+        query = """
+            SELECT * FROM file_optional
+            WHERE peer <= 10 AND %s
+            ORDER BY peer DESC, time_accessed, uploaded / size
+        """ % self.getOptionalUsedWhere()
+        limit_start = 0
+        while 1:
+            num = 0
+            res = self.execute("%s LIMIT %s, 50" % (query, limit_start))
+            for row in res:
+                yield row
+                num += 1
+            if num < 50:
+                break
+            limit_start += 50
+
+    def getOptionalLimitBytes(self):
+        if config.optional_limit.endswith("%"):
+            limit_percent = float(re.sub("[^0-9.]", "", config.optional_limit))
+            limit_bytes = helper.getFreeSpace() * (limit_percent / 100)
+        else:
+            limit_bytes = float(re.sub("[^0-9.]", "", config.optional_limit)) * 1024 * 1024 * 1024
+        return limit_bytes
+
+    def getOptionalUsedWhere(self):
+        maxsize = config.optional_limit_exclude_minsize * 1024 * 1024
+        query = "is_downloaded = 1 AND is_pinned = 0 AND size < %s" % maxsize
+
+        # Don't delete optional files from owned sites
+        my_site_ids = []
+        for address, site in self.sites.items():
+            if site.settings["own"]:
+                my_site_ids.append(str(self.site_ids[address]))
+
+        if my_site_ids:
+            query += " AND site_id NOT IN (%s)" % ", ".join(my_site_ids)
+        return query
+
+    def getOptionalUsedBytes(self):
+        size = self.execute("SELECT SUM(size) FROM file_optional WHERE %s" % self.getOptionalUsedWhere()).fetchone()[0]
+        if not size:
+            size = 0
+        return size
+
+    def getOptionalNeedDelete(self, size):
+        if config.optional_limit.endswith("%"):
+            limit_percent = float(re.sub("[^0-9.]", "", config.optional_limit))
+            need_delete = size - ((helper.getFreeSpace() + size) * (limit_percent / 100))
+        else:
+            need_delete = size - self.getOptionalLimitBytes()
+        return need_delete
+
+    def checkOptionalLimit(self, limit=None):
+        if not limit:
+            limit = self.getOptionalLimitBytes()
+
+        if limit < 0:
+            self.log.debug("Invalid limit for optional files: %s" % limit)
+            return False
+
+        size = self.getOptionalUsedBytes()
+
+        need_delete = self.getOptionalNeedDelete(size)
+
+        self.log.debug(
+            "Optional size: %.1fMB/%.1fMB, Need delete: %.1fMB" %
+            (float(size) / 1024 / 1024, float(limit) / 1024 / 1024, float(need_delete) / 1024 / 1024)
+        )
+        if need_delete <= 0:
+            return False
+
+        self.updatePeerNumbers()
+
+        site_ids_reverse = {val: key for key, val in self.site_ids.items()}
+        deleted_file_ids = []
+        for row in self.queryDeletableFiles():
+            site_address = site_ids_reverse.get(row["site_id"])
+            site = self.sites.get(site_address)
+            if not site:
+                self.log.error("No site found for id: %s" % row["site_id"])
+                continue
+            site.log.debug("Deleting %s %.3f MB left" % (row["inner_path"], float(need_delete) / 1024 / 1024))
+            deleted_file_ids.append(row["file_id"])
+            try:
+                site.content_manager.optionalRemoved(row["inner_path"], row["hash_id"], row["size"])
+                site.storage.delete(row["inner_path"])
+                need_delete -= row["size"]
+            except Exception as err:
+                site.log.error("Error deleting %s: %s" % (row["inner_path"], err))
+
+            if need_delete <= 0:
+                break
+
+        cur = self.getCursor()
+        for file_id in deleted_file_ids:
+            cur.execute("UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?", {"file_id": file_id})
+        cur.close()

plugins/OptionalManager/OptionalManagerPlugin.py

@@ -0,0 +1,239 @@
+import time
+import re
+import collections
+
+import gevent
+
+from util import helper
+from Plugin import PluginManager
+from . import ContentDbPlugin
+
+
+# We can only import plugin host clases after the plugins are loaded
+@PluginManager.afterLoad
+def importPluginnedClasses():
+    global config
+    from Config import config
+
+
+def processAccessLog():
+    if access_log:
+        content_db = ContentDbPlugin.content_db
+        now = int(time.time())
+        num = 0
+        for site_id in access_log:
+            content_db.execute(
+                "UPDATE file_optional SET time_accessed = %s WHERE ?" % now,
+                {"site_id": site_id, "inner_path": list(access_log[site_id].keys())}
+            )
+            num += len(access_log[site_id])
+        access_log.clear()
+
+
+def processRequestLog():
+    if request_log:
+        content_db = ContentDbPlugin.content_db
+        cur = content_db.getCursor()
+        num = 0
+        for site_id in request_log:
+            for inner_path, uploaded in request_log[site_id].items():
+                content_db.execute(
+                    "UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded,
+                    {"site_id": site_id, "inner_path": inner_path}
+                )
+                num += 1
+        request_log.clear()
+
+
+if "access_log" not in locals().keys():  # To keep between module reloads
+    access_log = collections.defaultdict(dict)  # {site_id: {inner_path1: 1, inner_path2: 1...}}
+    request_log = collections.defaultdict(lambda: collections.defaultdict(int))  # {site_id: {inner_path1: 1, inner_path2: 1...}}
+    helper.timer(61, processAccessLog)
+    helper.timer(60, processRequestLog)
+
+
+@PluginManager.registerTo("ContentManager")
+class ContentManagerPlugin(object):
+    def __init__(self, *args, **kwargs):
+        self.cache_is_pinned = {}
+        super(ContentManagerPlugin, self).__init__(*args, **kwargs)
+
+    def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):
+        if "|" in inner_path:  # Big file piece
+            file_inner_path, file_range = inner_path.split("|")
+        else:
+            file_inner_path = inner_path
+
+        self.contents.db.executeDelayed(
+            "UPDATE file_optional SET time_downloaded = :now, is_downloaded = 1, peer = peer + 1 WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 0",
+            {"now": int(time.time()), "site_id": self.contents.db.site_ids[self.site.address], "inner_path": file_inner_path}
+        )
+
+        return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own)
+
+    def optionalRemoved(self, inner_path, hash_id, size=None):
+        self.contents.db.execute(
+            "UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 1",
+            {"site_id": self.contents.db.site_ids[self.site.address], "inner_path": inner_path}
+        )
+
+        if self.contents.db.cur.cursor.rowcount > 0:
+            back = super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size)
+            # Re-add to hashfield if we have other file with the same hash_id
+            if self.isDownloaded(hash_id=hash_id, force_check_db=True):
+                self.hashfield.appendHashId(hash_id)
+        else:
+            back = False
+        self.cache_is_pinned = {}
+        return back
+
+    def optionalRenamed(self, inner_path_old, inner_path_new):
+        back = super(ContentManagerPlugin, self).optionalRenamed(inner_path_old, inner_path_new)
+        self.cache_is_pinned = {}
+        self.contents.db.execute(
+            "UPDATE file_optional SET inner_path = :inner_path_new WHERE site_id = :site_id AND inner_path = :inner_path_old",
+            {"site_id": self.contents.db.site_ids[self.site.address], "inner_path_old": inner_path_old, "inner_path_new": inner_path_new}
+        )
+        return back
+
+    def isDownloaded(self, inner_path=None, hash_id=None, force_check_db=False):
+        if hash_id and not force_check_db and hash_id not in self.hashfield:
+            return False
+
+        if inner_path:
+            res = self.contents.db.execute(
+                "SELECT is_downloaded FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1",
+                {"site_id": self.contents.db.site_ids[self.site.address], "inner_path": inner_path}
+            )
+        else:
+            res = self.contents.db.execute(
+                "SELECT is_downloaded FROM file_optional WHERE site_id = :site_id AND hash_id = :hash_id AND is_downloaded = 1 LIMIT 1",
+                {"site_id": self.contents.db.site_ids[self.site.address], "hash_id": hash_id}
+            )
+        row = res.fetchone()
+        if row and row["is_downloaded"]:
+            return True
+        else:
+            return False
+
+    def isPinned(self, inner_path):
+        if inner_path in self.cache_is_pinned:
+            self.site.log.debug("Cached is pinned: %s" % inner_path)
+            return self.cache_is_pinned[inner_path]
+
+        res = self.contents.db.execute(
+            "SELECT is_pinned FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1",
+            {"site_id": self.contents.db.site_ids[self.site.address], "inner_path": inner_path}
+        )
+        row = res.fetchone()
+
+        if row and row[0]:
+            is_pinned = True
+        else:
+            is_pinned = False
+
+        self.cache_is_pinned[inner_path] = is_pinned
+        self.site.log.debug("Cache set is pinned: %s %s" % (inner_path, is_pinned))
+
+        return is_pinned
+
+    def setPin(self, inner_path, is_pinned):
+        content_db = self.contents.db
+        site_id = content_db.site_ids[self.site.address]
+        content_db.execute("UPDATE file_optional SET is_pinned = %d WHERE ?" % is_pinned, {"site_id": site_id, "inner_path": inner_path})
+        self.cache_is_pinned = {}
+
+    def optionalDelete(self, inner_path):
+        if self.isPinned(inner_path):
+            self.site.log.debug("Skip deleting pinned optional file: %s" % inner_path)
+            return False
+        else:
+            return super(ContentManagerPlugin, self).optionalDelete(inner_path)
+
+
+@PluginManager.registerTo("WorkerManager")
+class WorkerManagerPlugin(object):
+    def doneTask(self, task):
+        super(WorkerManagerPlugin, self).doneTask(task)
+
+        if task["optional_hash_id"] and not self.tasks:  # Execute delayed queries immedietly after tasks finished
+            ContentDbPlugin.content_db.processDelayed()
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    def parsePath(self, path):
+        global access_log
+        path_parts = super(UiRequestPlugin, self).parsePath(path)
+        if path_parts:
+            site_id = ContentDbPlugin.content_db.site_ids.get(path_parts["request_address"])
+            if site_id:
+                if ContentDbPlugin.content_db.isOptionalFile(site_id, path_parts["inner_path"]):
+                    access_log[site_id][path_parts["inner_path"]] = 1
+        return path_parts
+
+
+@PluginManager.registerTo("FileRequest")
+class FileRequestPlugin(object):
+    def actionGetFile(self, params):
+        stats = super(FileRequestPlugin, self).actionGetFile(params)
+        self.recordFileRequest(params["site"], params["inner_path"], stats)
+        return stats
+
+    def actionStreamFile(self, params):
+        stats = super(FileRequestPlugin, self).actionStreamFile(params)
+        self.recordFileRequest(params["site"], params["inner_path"], stats)
+        return stats
+
+    def recordFileRequest(self, site_address, inner_path, stats):
+        if not stats:
+            # Only track the last request of files
+            return False
+        site_id = ContentDbPlugin.content_db.site_ids[site_address]
+        if site_id and ContentDbPlugin.content_db.isOptionalFile(site_id, inner_path):
+            request_log[site_id][inner_path] += stats["bytes_sent"]
+
+
+@PluginManager.registerTo("Site")
+class SitePlugin(object):
+    def isDownloadable(self, inner_path):
+        is_downloadable = super(SitePlugin, self).isDownloadable(inner_path)
+        if is_downloadable:
+            return is_downloadable
+
+        for path in self.settings.get("optional_help", {}).keys():
+            if inner_path.startswith(path):
+                return True
+
+        return False
+
+    def fileForgot(self, inner_path):
+        if "|" in inner_path and self.content_manager.isPinned(re.sub(r"\|.*", "", inner_path)):
+            self.log.debug("File %s is pinned, no fileForgot" % inner_path)
+            return False
+        else:
+            return super(SitePlugin, self).fileForgot(inner_path)
+
+    def fileDone(self, inner_path):
+        if "|" in inner_path and self.bad_files.get(inner_path, 0) > 5:  # Idle optional file done
+            inner_path_file = re.sub(r"\|.*", "", inner_path)
+            num_changed = 0
+            for key, val in self.bad_files.items():
+                if key.startswith(inner_path_file) and val > 1:
+                    self.bad_files[key] = 1
+                    num_changed += 1
+            self.log.debug("Idle optional file piece done, changed retry number of %s pieces." % num_changed)
+            if num_changed:
+                gevent.spawn(self.retryBadFiles)
+
+        return super(SitePlugin, self).fileDone(inner_path)
+
+
+@PluginManager.registerTo("ConfigPlugin")
+class ConfigPlugin(object):
+    def createArguments(self):
+        group = self.parser.add_argument_group("OptionalManager plugin")
+        group.add_argument('--optional_limit', help='Limit total size of optional files', default="10%", metavar="GB or free space %")
+        group.add_argument('--optional_limit_exclude_minsize', help='Exclude files larger than this limit from optional size limit calculation', default=20, metavar="MB", type=int)
+
+        return super(ConfigPlugin, self).createArguments()

plugins/OptionalManager/Test/TestOptionalManager.py

@@ -0,0 +1,158 @@
+import copy
+
+import pytest
+
+
+@pytest.mark.usefixtures("resetSettings")
+class TestOptionalManager:
+    def testDbFill(self, site):
+        contents = site.content_manager.contents
+        assert len(site.content_manager.hashfield) > 0
+        assert contents.db.execute("SELECT COUNT(*) FROM file_optional WHERE is_downloaded = 1").fetchone()[0] == len(site.content_manager.hashfield)
+
+    def testSetContent(self, site):
+        contents = site.content_manager.contents
+
+        # Add new file
+        new_content = copy.deepcopy(contents["content.json"])
+        new_content["files_optional"]["testfile"] = {
+            "size": 1234,
+            "sha512": "aaaabbbbcccc"
+        }
+        num_optional_files_before = contents.db.execute("SELECT COUNT(*) FROM file_optional").fetchone()[0]
+        contents["content.json"] = new_content
+        assert contents.db.execute("SELECT COUNT(*) FROM file_optional").fetchone()[0] > num_optional_files_before
+
+        # Remove file
+        new_content = copy.deepcopy(contents["content.json"])
+        del new_content["files_optional"]["testfile"]
+        num_optional_files_before = contents.db.execute("SELECT COUNT(*) FROM file_optional").fetchone()[0]
+        contents["content.json"] = new_content
+        assert contents.db.execute("SELECT COUNT(*) FROM file_optional").fetchone()[0] < num_optional_files_before
+
+    def testDeleteContent(self, site):
+        contents = site.content_manager.contents
+        num_optional_files_before = contents.db.execute("SELECT COUNT(*) FROM file_optional").fetchone()[0]
+        del contents["content.json"]
+        assert contents.db.execute("SELECT COUNT(*) FROM file_optional").fetchone()[0] < num_optional_files_before
+
+    def testVerifyFiles(self, site):
+        contents = site.content_manager.contents
+
+        # Add new file
+        new_content = copy.deepcopy(contents["content.json"])
+        new_content["files_optional"]["testfile"] = {
+            "size": 1234,
+            "sha512": "aaaabbbbcccc"
+        }
+        contents["content.json"] = new_content
+        file_row = contents.db.execute("SELECT * FROM file_optional WHERE inner_path = 'testfile'").fetchone()
+        assert not file_row["is_downloaded"]
+
+        # Write file from outside of ZeroNet
+        site.storage.open("testfile", "wb").write(b"A" * 1234)  # For quick check hash does not matter only file size
+
+        hashfield_len_before = len(site.content_manager.hashfield)
+        site.storage.verifyFiles(quick_check=True)
+        assert len(site.content_manager.hashfield) == hashfield_len_before + 1
+
+        file_row = contents.db.execute("SELECT * FROM file_optional WHERE inner_path = 'testfile'").fetchone()
+        assert file_row["is_downloaded"]
+
+        # Delete file outside of ZeroNet
+        site.storage.delete("testfile")
+        site.storage.verifyFiles(quick_check=True)
+        file_row = contents.db.execute("SELECT * FROM file_optional WHERE inner_path = 'testfile'").fetchone()
+        assert not file_row["is_downloaded"]
+
+    def testVerifyFilesSameHashId(self, site):
+        contents = site.content_manager.contents
+
+        new_content = copy.deepcopy(contents["content.json"])
+
+        # Add two files with same hashid (first 4 character)
+        new_content["files_optional"]["testfile1"] = {
+            "size": 1234,
+            "sha512": "aaaabbbbcccc"
+        }
+        new_content["files_optional"]["testfile2"] = {
+            "size": 2345,
+            "sha512": "aaaabbbbdddd"
+        }
+        contents["content.json"] = new_content
+
+        assert site.content_manager.hashfield.getHashId("aaaabbbbcccc") == site.content_manager.hashfield.getHashId("aaaabbbbdddd")
+
+        # Write files from outside of ZeroNet (For quick check hash does not matter only file size)
+        site.storage.open("testfile1", "wb").write(b"A" * 1234)
+        site.storage.open("testfile2", "wb").write(b"B" * 2345)
+
+        site.storage.verifyFiles(quick_check=True)
+
+        # Make sure that both is downloaded
+        assert site.content_manager.isDownloaded("testfile1")
+        assert site.content_manager.isDownloaded("testfile2")
+        assert site.content_manager.hashfield.getHashId("aaaabbbbcccc") in site.content_manager.hashfield
+
+        # Delete one of the files
+        site.storage.delete("testfile1")
+        site.storage.verifyFiles(quick_check=True)
+        assert not site.content_manager.isDownloaded("testfile1")
+        assert site.content_manager.isDownloaded("testfile2")
+        assert site.content_manager.hashfield.getHashId("aaaabbbbdddd") in site.content_manager.hashfield
+
+    def testIsPinned(self, site):
+        assert not site.content_manager.isPinned("data/img/zerotalk-upvote.png")
+        site.content_manager.setPin("data/img/zerotalk-upvote.png", True)
+        assert site.content_manager.isPinned("data/img/zerotalk-upvote.png")
+
+        assert len(site.content_manager.cache_is_pinned) == 1
+        site.content_manager.cache_is_pinned = {}
+        assert site.content_manager.isPinned("data/img/zerotalk-upvote.png")
+
+    def testBigfilePieceReset(self, site):
+        site.bad_files = {
+            "data/fake_bigfile.mp4|0-1024": 10,
+            "data/fake_bigfile.mp4|1024-2048": 10,
+            "data/fake_bigfile.mp4|2048-3064": 10
+        }
+        site.onFileDone("data/fake_bigfile.mp4|0-1024")
+        assert site.bad_files["data/fake_bigfile.mp4|1024-2048"] == 1
+        assert site.bad_files["data/fake_bigfile.mp4|2048-3064"] == 1
+
+    def testOptionalDelete(self, site):
+        contents = site.content_manager.contents
+
+        site.content_manager.setPin("data/img/zerotalk-upvote.png", True)
+        site.content_manager.setPin("data/img/zeroid.png", False)
+        new_content = copy.deepcopy(contents["content.json"])
+        del new_content["files_optional"]["data/img/zerotalk-upvote.png"]
+        del new_content["files_optional"]["data/img/zeroid.png"]
+
+        assert site.storage.isFile("data/img/zerotalk-upvote.png")
+        assert site.storage.isFile("data/img/zeroid.png")
+
+        site.storage.writeJson("content.json", new_content)
+        site.content_manager.loadContent("content.json", force=True)
+
+        assert not site.storage.isFile("data/img/zeroid.png")
+        assert site.storage.isFile("data/img/zerotalk-upvote.png")
+
+    def testOptionalRename(self, site):
+        contents = site.content_manager.contents
+
+        site.content_manager.setPin("data/img/zerotalk-upvote.png", True)
+        new_content = copy.deepcopy(contents["content.json"])
+        new_content["files_optional"]["data/img/zerotalk-upvote-new.png"] = new_content["files_optional"]["data/img/zerotalk-upvote.png"]
+        del new_content["files_optional"]["data/img/zerotalk-upvote.png"]
+
+        assert site.storage.isFile("data/img/zerotalk-upvote.png")
+        assert site.content_manager.isPinned("data/img/zerotalk-upvote.png")
+
+        site.storage.writeJson("content.json", new_content)
+        site.content_manager.loadContent("content.json", force=True)
+
+        assert not site.storage.isFile("data/img/zerotalk-upvote.png")
+        assert not site.content_manager.isPinned("data/img/zerotalk-upvote.png")
+        assert site.content_manager.isPinned("data/img/zerotalk-upvote-new.png")
+        assert site.storage.isFile("data/img/zerotalk-upvote-new.png")

plugins/OptionalManager/Test/conftest.py

@@ -0,0 +1 @@
+from src.Test.conftest import *

plugins/OptionalManager/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/OptionalManager/UiWebsocketPlugin.py

@@ -0,0 +1,382 @@
+import re
+import time
+import html
+
+import gevent
+
+from Plugin import PluginManager
+from Config import config
+from util import helper
+from Translate import Translate
+
+if "_" not in locals():
+    _ = Translate("plugins/OptionalManager/languages/")
+
+bigfile_sha512_cache = {}
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def __init__(self, *args, **kwargs):
+        self.time_peer_numbers_updated = 0
+        super(UiWebsocketPlugin, self).__init__(*args, **kwargs)
+
+    def actionSiteSign(self, to, privatekey=None, inner_path="content.json", *args, **kwargs):
+        # Add file to content.db and set it as pinned
+        content_db = self.site.content_manager.contents.db
+        content_inner_dir = helper.getDirname(inner_path)
+        content_db.my_optional_files[self.site.address + "/" + content_inner_dir] = time.time()
+        if len(content_db.my_optional_files) > 50:  # Keep only last 50
+            oldest_key = min(
+                iter(content_db.my_optional_files.keys()),
+                key=(lambda key: content_db.my_optional_files[key])
+            )
+            del content_db.my_optional_files[oldest_key]
+
+        return super(UiWebsocketPlugin, self).actionSiteSign(to, privatekey, inner_path, *args, **kwargs)
+
+    def updatePeerNumbers(self):
+        self.site.updateHashfield()
+        content_db = self.site.content_manager.contents.db
+        content_db.updatePeerNumbers()
+        self.site.updateWebsocket(peernumber_updated=True)
+
+    def addBigfileInfo(self, row):
+        global bigfile_sha512_cache
+
+        content_db = self.site.content_manager.contents.db
+        site = content_db.sites[row["address"]]
+        if not site.settings.get("has_bigfile"):
+            return False
+
+        file_key = row["address"] + "/" + row["inner_path"]
+        sha512 = bigfile_sha512_cache.get(file_key)
+        file_info = None
+        if not sha512:
+            file_info = site.content_manager.getFileInfo(row["inner_path"])
+            if not file_info or not file_info.get("piece_size"):
+                return False
+            sha512 = file_info["sha512"]
+            bigfile_sha512_cache[file_key] = sha512
+
+        if sha512 in site.storage.piecefields:
+            piecefield = site.storage.piecefields[sha512].tobytes()
+        else:
+            piecefield = None
+
+        if piecefield:
+            row["pieces"] = len(piecefield)
+            row["pieces_downloaded"] = piecefield.count(b"\x01")
+            row["downloaded_percent"] = 100 * row["pieces_downloaded"] / row["pieces"]
+            if row["pieces_downloaded"]:
+                if row["pieces"] == row["pieces_downloaded"]:
+                    row["bytes_downloaded"] = row["size"]
+                else:
+                    if not file_info:
+                        file_info = site.content_manager.getFileInfo(row["inner_path"])
+                    row["bytes_downloaded"] = row["pieces_downloaded"] * file_info.get("piece_size", 0)
+            else:
+                row["bytes_downloaded"] = 0
+
+            row["is_downloading"] = bool(next((inner_path for inner_path in site.bad_files if inner_path.startswith(row["inner_path"])), False))
+
+        # Add leech / seed stats
+        row["peer_seed"] = 0
+        row["peer_leech"] = 0
+        for peer in site.peers.values():
+            if not peer.time_piecefields_updated or sha512 not in peer.piecefields:
+                continue
+            peer_piecefield = peer.piecefields[sha512].tobytes()
+            if not peer_piecefield:
+                continue
+            if peer_piecefield == b"\x01" * len(peer_piecefield):
+                row["peer_seed"] += 1
+            else:
+                row["peer_leech"] += 1
+
+        # Add myself
+        if piecefield:
+            if row["pieces_downloaded"] == row["pieces"]:
+                row["peer_seed"] += 1
+            else:
+                row["peer_leech"] += 1
+
+        return True
+
+    # Optional file functions
+
+    def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC", limit=10, filter="downloaded"):
+        if not address:
+            address = self.site.address
+
+        # Update peer numbers if necessary
+        content_db = self.site.content_manager.contents.db
+        if time.time() - content_db.time_peer_numbers_updated > 60 * 1 and time.time() - self.time_peer_numbers_updated > 60 * 5:
+            # Start in new thread to avoid blocking
+            self.time_peer_numbers_updated = time.time()
+            gevent.spawn(self.updatePeerNumbers)
+
+        if address == "all" and "ADMIN" not in self.permissions:
+            return self.response(to, {"error": "Forbidden"})
+
+        if not self.hasSitePermission(address):
+            return self.response(to, {"error": "Forbidden"})
+
+        if not all([re.match("^[a-z_*/+-]+( DESC| ASC|)$", part.strip()) for part in orderby.split(",")]):
+            return self.response(to, "Invalid order_by")
+
+        if type(limit) != int:
+            return self.response(to, "Invalid limit")
+
+        back = []
+        content_db = self.site.content_manager.contents.db
+
+        wheres = {}
+        wheres_raw = []
+        if "bigfile" in filter:
+            wheres["size >"] = 1024 * 1024 * 10
+        if "downloaded" in filter:
+            wheres_raw.append("(is_downloaded = 1 OR is_pinned = 1)")
+        if "pinned" in filter:
+            wheres["is_pinned"] = 1
+
+        if address == "all":
+            join = "LEFT JOIN site USING (site_id)"
+        else:
+            wheres["site_id"] = content_db.site_ids[address]
+            join = ""
+
+        if wheres_raw:
+            query_wheres_raw = "AND" + " AND ".join(wheres_raw)
+        else:
+            query_wheres_raw = ""
+
+        query = "SELECT * FROM file_optional %s WHERE ? %s ORDER BY %s LIMIT %s" % (join, query_wheres_raw, orderby, limit)
+
+        for row in content_db.execute(query, wheres):
+            row = dict(row)
+            if address != "all":
+                row["address"] = address
+
+            if row["size"] > 1024 * 1024:
+                has_info = self.addBigfileInfo(row)
+            else:
+                has_info = False
+
+            if not has_info:
+                if row["is_downloaded"]:
+                    row["bytes_downloaded"] = row["size"]
+                    row["downloaded_percent"] = 100
+                else:
+                    row["bytes_downloaded"] = 0
+                    row["downloaded_percent"] = 0
+
+            back.append(row)
+        self.response(to, back)
+
+    def actionOptionalFileInfo(self, to, inner_path):
+        content_db = self.site.content_manager.contents.db
+        site_id = content_db.site_ids[self.site.address]
+
+        # Update peer numbers if necessary
+        if time.time() - content_db.time_peer_numbers_updated > 60 * 1 and time.time() - self.time_peer_numbers_updated > 60 * 5:
+            # Start in new thread to avoid blocking
+            self.time_peer_numbers_updated = time.time()
+            gevent.spawn(self.updatePeerNumbers)
+
+        query = "SELECT * FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1"
+        res = content_db.execute(query, {"site_id": site_id, "inner_path": inner_path})
+        row = next(res, None)
+        if row:
+            row = dict(row)
+            if row["size"] > 1024 * 1024:
+                row["address"] = self.site.address
+                self.addBigfileInfo(row)
+            self.response(to, row)
+        else:
+            self.response(to, None)
+
+    def setPin(self, inner_path, is_pinned, address=None):
+        if not address:
+            address = self.site.address
+
+        if not self.hasSitePermission(address):
+            return {"error": "Forbidden"}
+
+        site = self.server.sites[address]
+        site.content_manager.setPin(inner_path, is_pinned)
+
+        return "ok"
+
+    def actionOptionalFilePin(self, to, inner_path, address=None):
+        if type(inner_path) is not list:
+            inner_path = [inner_path]
+        back = self.setPin(inner_path, 1, address)
+        num_file = len(inner_path)
+        if back == "ok":
+            if num_file == 1:
+                self.cmd("notification", ["done", _["Pinned %s"] % html.escape(helper.getFilename(inner_path[0])), 5000])
+            else:
+                self.cmd("notification", ["done", _["Pinned %s files"] % num_file, 5000])
+        self.response(to, back)
+
+    def actionOptionalFileUnpin(self, to, inner_path, address=None):
+        if type(inner_path) is not list:
+            inner_path = [inner_path]
+        back = self.setPin(inner_path, 0, address)
+        num_file = len(inner_path)
+        if back == "ok":
+            if num_file == 1:
+                self.cmd("notification", ["done", _["Removed pin from %s"] % html.escape(helper.getFilename(inner_path[0])), 5000])
+            else:
+                self.cmd("notification", ["done", _["Removed pin from %s files"] % num_file, 5000])
+        self.response(to, back)
+
+    def actionOptionalFileDelete(self, to, inner_path, address=None):
+        if not address:
+            address = self.site.address
+
+        if not self.hasSitePermission(address):
+            return self.response(to, {"error": "Forbidden"})
+
+        site = self.server.sites[address]
+
+        content_db = site.content_manager.contents.db
+        site_id = content_db.site_ids[site.address]
+
+        res = content_db.execute("SELECT * FROM file_optional WHERE ? LIMIT 1", {"site_id": site_id, "inner_path": inner_path, "is_downloaded": 1})
+        row = next(res, None)
+
+        if not row:
+            return self.response(to, {"error": "Not found in content.db"})
+
+        removed = site.content_manager.optionalRemoved(inner_path, row["hash_id"], row["size"])
+        # if not removed:
+        #    return self.response(to, {"error": "Not found in hash_id: %s" % row["hash_id"]})
+
+        content_db.execute("UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?", {"site_id": site_id, "inner_path": inner_path})
+
+        try:
+            site.storage.delete(inner_path)
+        except Exception as err:
+            return self.response(to, {"error": "File delete error: %s" % err})
+        site.updateWebsocket(file_delete=inner_path)
+
+        if inner_path in site.content_manager.cache_is_pinned:
+            site.content_manager.cache_is_pinned = {}
+
+        self.response(to, "ok")
+
+    # Limit functions
+
+    def actionOptionalLimitStats(self, to):
+        if "ADMIN" not in self.site.settings["permissions"]:
+            return self.response(to, "Forbidden")
+
+        back = {}
+        back["limit"] = config.optional_limit
+        back["used"] = self.site.content_manager.contents.db.getOptionalUsedBytes()
+        back["free"] = helper.getFreeSpace()
+
+        self.response(to, back)
+
+    def actionOptionalLimitSet(self, to, limit):
+        if "ADMIN" not in self.site.settings["permissions"]:
+            return self.response(to, {"error": "Forbidden"})
+        config.optional_limit = re.sub("\.0+$", "", limit)  # Remove unnecessary digits from end
+        config.saveValue("optional_limit", limit)
+        self.response(to, "ok")
+
+    # Distribute help functions
+
+    def actionOptionalHelpList(self, to, address=None):
+        if not address:
+            address = self.site.address
+
+        if not self.hasSitePermission(address):
+            return self.response(to, {"error": "Forbidden"})
+
+        site = self.server.sites[address]
+
+        self.response(to, site.settings.get("optional_help", {}))
+
+    def actionOptionalHelp(self, to, directory, title, address=None):
+        if not address:
+            address = self.site.address
+
+        if not self.hasSitePermission(address):
+            return self.response(to, {"error": "Forbidden"})
+
+        site = self.server.sites[address]
+        content_db = site.content_manager.contents.db
+        site_id = content_db.site_ids[address]
+
+        if "optional_help" not in site.settings:
+            site.settings["optional_help"] = {}
+
+        stats = content_db.execute(
+            "SELECT COUNT(*) AS num, SUM(size) AS size FROM file_optional WHERE site_id = :site_id AND inner_path LIKE :inner_path",
+            {"site_id": site_id, "inner_path": directory + "%"}
+        ).fetchone()
+        stats = dict(stats)
+
+        if not stats["size"]:
+            stats["size"] = 0
+        if not stats["num"]:
+            stats["num"] = 0
+
+        self.cmd("notification", [
+            "done",
+            _["You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>"] %
+            (html.escape(title), html.escape(directory)),
+            10000
+        ])
+
+        site.settings["optional_help"][directory] = title
+
+        self.response(to, dict(stats))
+
+    def actionOptionalHelpRemove(self, to, directory, address=None):
+        if not address:
+            address = self.site.address
+
+        if not self.hasSitePermission(address):
+            return self.response(to, {"error": "Forbidden"})
+
+        site = self.server.sites[address]
+
+        try:
+            del site.settings["optional_help"][directory]
+            self.response(to, "ok")
+        except Exception:
+            self.response(to, {"error": "Not found"})
+
+    def cbOptionalHelpAll(self, to, site, value):
+        site.settings["autodownloadoptional"] = value
+        self.response(to, value)
+
+    def actionOptionalHelpAll(self, to, value, address=None):
+        if not address:
+            address = self.site.address
+
+        if not self.hasSitePermission(address):
+            return self.response(to, {"error": "Forbidden"})
+
+        site = self.server.sites[address]
+
+        if value:
+            if "ADMIN" in self.site.settings["permissions"]:
+                self.cbOptionalHelpAll(to, site, True)
+            else:
+                site_title = site.content_manager.contents["content.json"].get("title", address)
+                self.cmd(
+                    "confirm",
+                    [
+                        _["Help distribute all new optional files on site <b>%s</b>"] % html.escape(site_title),
+                        _["Yes, I want to help!"]
+                    ],
+                    lambda res: self.cbOptionalHelpAll(to, site, True)
+                )
+        else:
+            site.settings["autodownloadoptional"] = False
+            self.response(to, False)

plugins/OptionalManager/__init__.py

@@ -0,0 +1 @@
+from . import OptionalManagerPlugin

plugins/OptionalManager/languages/es.json

@@ -0,0 +1,7 @@
+{
+	"Pinned %s files": "Archivos %s fijados",
+	"Removed pin from %s files": "Archivos %s que no estan fijados",
+	"You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>": "Tu empezaste a ayudar a distribuir <b>%s</b>.<br><small>Directorio: %s</small>",
+	"Help distribute all new optional files on site <b>%s</b>": "Ayude a distribuir todos los archivos opcionales en el sitio <b>%s</b>",
+	"Yes, I want to help!": "¡Si, yo quiero ayudar!"
+}

plugins/OptionalManager/languages/fr.json

@@ -0,0 +1,7 @@
+{
+	"Pinned %s files": "Fichiers %s épinglés",
+	"Removed pin from %s files": "Fichiers %s ne sont plus épinglés",
+	"You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>": "Vous avez commencé à aider à distribuer <b>%s</b>.<br><small>Dossier : %s</small>",
+	"Help distribute all new optional files on site <b>%s</b>": "Aider à distribuer tous les fichiers optionnels du site <b>%s</b>",
+	"Yes, I want to help!": "Oui, je veux aider !"
+}

plugins/OptionalManager/languages/hu.json

@@ -0,0 +1,7 @@
+{
+	"Pinned %s files": "%s fájl rögzítve",
+	"Removed pin from %s files": "%s fájl rögzítés eltávolítva",
+	"You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>": "Új segítség a terjesztésben: <b>%s</b>.<br><small>Könyvtár: %s</small>",
+	"Help distribute all new optional files on site <b>%s</b>": "Segítség az összes új opcionális fájl terjesztésében az <b>%s</b> oldalon",
+	"Yes, I want to help!": "Igen, segíteni akarok!"
+}

plugins/OptionalManager/languages/pt-br.json

@@ -0,0 +1,7 @@
+{
+	"Pinned %s files": "Arquivos %s fixados",
+	"Removed pin from %s files": "Arquivos %s não estão fixados",
+	"You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>": "Você começou a ajudar a distribuir <b>%s</b>.<br><small>Pasta: %s</small>",
+	"Help distribute all new optional files on site <b>%s</b>": "Ajude a distribuir todos os novos arquivos opcionais no site <b>%s</b>",
+	"Yes, I want to help!": "Sim, eu quero ajudar!"
+}

plugins/OptionalManager/languages/zh-tw.json

@@ -0,0 +1,7 @@
+{
+	"Pinned %s files": "已固定 %s 個檔",
+	"Removed pin from %s files": "已解除固定 %s 個檔",
+	"You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>": "你已經開始幫助分發 <b>%s</b> 。<br><small>目錄：%s</small>",
+	"Help distribute all new optional files on site <b>%s</b>": "你想要幫助分發 <b>%s</b> 網站的所有檔嗎？",
+	"Yes, I want to help!": "是，我想要幫助！"
+}

plugins/OptionalManager/languages/zh.json

@@ -0,0 +1,7 @@
+{
+	"Pinned %s files": "已固定 %s 个文件",
+	"Removed pin from %s files": "已解除固定 %s 个文件",
+	"You started to help distribute <b>%s</b>.<br><small>Directory: %s</small>": "您已经开始帮助分发 <b>%s</b> 。<br><small>目录：%s</small>",
+	"Help distribute all new optional files on site <b>%s</b>": "您想要帮助分发 <b>%s</b> 站点的所有文件吗？",
+	"Yes, I want to help!": "是，我想要帮助！"
+}

plugins/PeerDb/PeerDbPlugin.py

@@ -0,0 +1,100 @@
+import time
+import sqlite3
+import random
+import atexit
+
+import gevent
+from Plugin import PluginManager
+
+
+@PluginManager.registerTo("ContentDb")
+class ContentDbPlugin(object):
+    def __init__(self, *args, **kwargs):
+        atexit.register(self.saveAllPeers)
+        super(ContentDbPlugin, self).__init__(*args, **kwargs)
+
+    def getSchema(self):
+        schema = super(ContentDbPlugin, self).getSchema()
+
+        schema["tables"]["peer"] = {
+            "cols": [
+                ["site_id", "INTEGER REFERENCES site (site_id) ON DELETE CASCADE"],
+                ["address", "TEXT NOT NULL"],
+                ["port", "INTEGER NOT NULL"],
+                ["hashfield", "BLOB"],
+                ["reputation", "INTEGER NOT NULL"],
+                ["time_added", "INTEGER NOT NULL"],
+                ["time_found", "INTEGER NOT NULL"]
+            ],
+            "indexes": [
+                "CREATE UNIQUE INDEX peer_key ON peer (site_id, address, port)"
+            ],
+            "schema_changed": 2
+        }
+
+        return schema
+
+    def loadPeers(self, site):
+        s = time.time()
+        site_id = self.site_ids.get(site.address)
+        res = self.execute("SELECT * FROM peer WHERE site_id = :site_id", {"site_id": site_id})
+        num = 0
+        num_hashfield = 0
+        for row in res:
+            peer = site.addPeer(str(row["address"]), row["port"])
+            if not peer:  # Already exist
+                continue
+            if row["hashfield"]:
+                peer.hashfield.replaceFromBytes(row["hashfield"])
+                num_hashfield += 1
+            peer.time_added = row["time_added"]
+            peer.time_found = row["time_found"]
+            peer.reputation = row["reputation"]
+            if row["address"].endswith(".onion"):
+                peer.reputation = peer.reputation / 2 - 1 # Onion peers less likely working
+            num += 1
+        if num_hashfield:
+            site.content_manager.has_optional_files = True
+        site.log.debug("%s peers (%s with hashfield) loaded in %.3fs" % (num, num_hashfield, time.time() - s))
+
+    def iteratePeers(self, site):
+        site_id = self.site_ids.get(site.address)
+        for key, peer in site.peers.items():
+            address, port = key.rsplit(":", 1)
+            if peer.has_hashfield:
+                hashfield = sqlite3.Binary(peer.hashfield.tobytes())
+            else:
+                hashfield = ""
+            yield (site_id, address, port, hashfield, peer.reputation, int(peer.time_added), int(peer.time_found))
+
+    def savePeers(self, site, spawn=False):
+        if spawn:
+            # Save peers every hour (+random some secs to not update very site at same time)
+            gevent.spawn_later(60 * 60 + random.randint(0, 60), self.savePeers, site, spawn=True)
+        if not site.peers:
+            site.log.debug("Peers not saved: No peers found")
+            return
+        s = time.time()
+        site_id = self.site_ids.get(site.address)
+        cur = self.getCursor()
+        try:
+            cur.execute("DELETE FROM peer WHERE site_id = :site_id", {"site_id": site_id})
+            cur.cursor.executemany(
+                "INSERT INTO peer (site_id, address, port, hashfield, reputation, time_added, time_found) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                self.iteratePeers(site)
+            )
+        except Exception as err:
+            site.log.error("Save peer error: %s" % err)
+        site.log.debug("Peers saved in %.3fs" % (time.time() - s))
+
+    def initSite(self, site):
+        super(ContentDbPlugin, self).initSite(site)
+        gevent.spawn_later(0.5, self.loadPeers, site)
+        gevent.spawn_later(60*60, self.savePeers, site, spawn=True)
+
+    def saveAllPeers(self):
+        for site in list(self.sites.values()):
+            try:
+                self.savePeers(site)
+            except Exception as err:
+                site.log.error("Save peer error: %s" % err)

plugins/PeerDb/__init__.py

@@ -0,0 +1,2 @@
+from . import PeerDbPlugin
+

plugins/Sidebar/SidebarPlugin.py

@@ -0,0 +1,794 @@
+import re
+import os
+import html
+import sys
+import math
+import time
+import json
+import io
+import urllib
+import urllib.parse
+
+import gevent
+
+from Config import config
+from Plugin import PluginManager
+from Debug import Debug
+from Translate import Translate
+from util import helper
+from .ZipStream import ZipStream
+
+plugin_dir = "plugins/Sidebar"
+media_dir = plugin_dir + "/media"
+sys.path.append(plugin_dir)  # To able to load geoip lib
+
+loc_cache = {}
+if "_" not in locals():
+    _ = Translate(plugin_dir + "/languages/")
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+    # Inject our resources to end of original file streams
+    def actionUiMedia(self, path):
+        if path == "/uimedia/all.js" or path == "/uimedia/all.css":
+            # First yield the original file and header
+            body_generator = super(UiRequestPlugin, self).actionUiMedia(path)
+            for part in body_generator:
+                yield part
+
+            # Append our media file to the end
+            ext = re.match(".*(js|css)$", path).group(1)
+            plugin_media_file = "%s/all.%s" % (media_dir, ext)
+            if config.debug:
+                # If debugging merge *.css to all.css and *.js to all.js
+                from Debug import DebugMedia
+                DebugMedia.merge(plugin_media_file)
+            if ext == "js":
+                yield _.translateData(open(plugin_media_file).read()).encode("utf8")
+            else:
+                for part in self.actionFile(plugin_media_file, send_header=False):
+                    yield part
+        elif path.startswith("/uimedia/globe/"):  # Serve WebGL globe files
+            file_name = re.match(".*/(.*)", path).group(1)
+            plugin_media_file = "%s_globe/%s" % (media_dir, file_name)
+            if config.debug and path.endswith("all.js"):
+                # If debugging merge *.css to all.css and *.js to all.js
+                from Debug import DebugMedia
+                DebugMedia.merge(plugin_media_file)
+            for part in self.actionFile(plugin_media_file):
+                yield part
+        else:
+            for part in super(UiRequestPlugin, self).actionUiMedia(path):
+                yield part
+
+    def actionZip(self):
+        address = self.get["address"]
+        site = self.server.site_manager.get(address)
+        if not site:
+            return self.error404("Site not found")
+
+        title = site.content_manager.contents.get("content.json", {}).get("title", "")
+        filename = "%s-backup-%s.zip" % (title, time.strftime("%Y-%m-%d_%H_%M"))
+        filename_quoted = urllib.parse.quote(filename)
+        self.sendHeader(content_type="application/zip", extra_headers={'Content-Disposition': 'attachment; filename="%s"' % filename_quoted})
+
+        return self.streamZip(site.storage.getPath("."))
+
+    def streamZip(self, dir_path):
+        zs = ZipStream(dir_path)
+        while 1:
+            data = zs.read()
+            if not data:
+                break
+            yield data
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def __init__(self, *args, **kwargs):
+        self.async_commands.add("sidebarGetPeers")
+        return super(UiWebsocketPlugin, self).__init__(*args, **kwargs)
+
+    def sidebarRenderPeerStats(self, body, site):
+        connected = len([peer for peer in list(site.peers.values()) if peer.connection and peer.connection.connected])
+        connectable = len([peer_id for peer_id in list(site.peers.keys()) if not peer_id.endswith(":0")])
+        onion = len([peer_id for peer_id in list(site.peers.keys()) if ".onion" in peer_id])
+        local = len([peer for peer in list(site.peers.values()) if helper.isPrivateIp(peer.ip)])
+        peers_total = len(site.peers)
+
+        # Add myself
+        if site.isServing():
+            peers_total += 1
+            if any(site.connection_server.port_opened.values()):
+                connectable += 1
+            if site.connection_server.tor_manager.start_onions:
+                onion += 1
+
+        if peers_total:
+            percent_connected = float(connected) / peers_total
+            percent_connectable = float(connectable) / peers_total
+            percent_onion = float(onion) / peers_total
+        else:
+            percent_connectable = percent_connected = percent_onion = 0
+
+        if local:
+            local_html = _("<li class='color-yellow'><span>{_[Local]}:</span><b>{local}</b></li>")
+        else:
+            local_html = ""
+
+        peer_ips = [peer.key for peer in site.getConnectablePeers(20, allow_private=False)]
+        peer_ips.sort(key=lambda peer_ip: ".onion:" in peer_ip)
+        copy_link = "http://127.0.0.1:43110/%s/?zeronet_peers=%s" % (
+            site.content_manager.contents["content.json"].get("domain", site.address),
+            ",".join(peer_ips)
+        )
+
+        body.append(_("""
+            <li>
+             <label>
+              {_[Peers]}
+              <small class="label-right"><a href='{copy_link}' id='link-copypeers' class='link-right'>{_[Copy to clipboard]}</a></small>
+             </label>
+             <ul class='graph'>
+              <li style='width: 100%' class='total back-black' title="{_[Total peers]}"></li>
+              <li style='width: {percent_connectable:.0%}' class='connectable back-blue' title='{_[Connectable peers]}'></li>
+              <li style='width: {percent_onion:.0%}' class='connected back-purple' title='{_[Onion]}'></li>
+              <li style='width: {percent_connected:.0%}' class='connected back-green' title='{_[Connected peers]}'></li>
+             </ul>
+             <ul class='graph-legend'>
+              <li class='color-green'><span>{_[Connected]}:</span><b>{connected}</b></li>
+              <li class='color-blue'><span>{_[Connectable]}:</span><b>{connectable}</b></li>
+              <li class='color-purple'><span>{_[Onion]}:</span><b>{onion}</b></li>
+              {local_html}
+              <li class='color-black'><span>{_[Total]}:</span><b>{peers_total}</b></li>
+             </ul>
+            </li>
+        """.replace("{local_html}", local_html)))
+
+    def sidebarRenderTransferStats(self, body, site):
+        recv = float(site.settings.get("bytes_recv", 0)) / 1024 / 1024
+        sent = float(site.settings.get("bytes_sent", 0)) / 1024 / 1024
+        transfer_total = recv + sent
+        if transfer_total:
+            percent_recv = recv / transfer_total
+            percent_sent = sent / transfer_total
+        else:
+            percent_recv = 0.5
+            percent_sent = 0.5
+
+        body.append(_("""
+            <li>
+             <label>{_[Data transfer]}</label>
+             <ul class='graph graph-stacked'>
+              <li style='width: {percent_recv:.0%}' class='received back-yellow' title="{_[Received bytes]}"></li>
+              <li style='width: {percent_sent:.0%}' class='sent back-green' title="{_[Sent bytes]}"></li>
+             </ul>
+             <ul class='graph-legend'>
+              <li class='color-yellow'><span>{_[Received]}:</span><b>{recv:.2f}MB</b></li>
+              <li class='color-green'<span>{_[Sent]}:</span><b>{sent:.2f}MB</b></li>
+             </ul>
+            </li>
+        """))
+
+    def sidebarRenderFileStats(self, body, site):
+        body.append(_("""
+            <li>
+             <label>
+              {_[Files]}
+              <small class="label-right"><a href='#Site+directory' id='link-directory' class='link-right'>{_[Open site directory]}</a>
+              <a href='/ZeroNet-Internal/Zip?address={site.address}' id='link-zip' class='link-right' download='site.zip'>{_[Save as .zip]}</a></small>
+             </label>
+             <ul class='graph graph-stacked'>
+        """))
+
+        extensions = (
+            ("html", "yellow"),
+            ("css", "orange"),
+            ("js", "purple"),
+            ("Image", "green"),
+            ("json", "darkblue"),
+            ("User data", "blue"),
+            ("Other", "white"),
+            ("Total", "black")
+        )
+        # Collect stats
+        size_filetypes = {}
+        size_total = 0
+        contents = site.content_manager.listContents()  # Without user files
+        for inner_path in contents:
+            content = site.content_manager.contents[inner_path]
+            if "files" not in content or content["files"] is None:
+                continue
+            for file_name, file_details in list(content["files"].items()):
+                size_total += file_details["size"]
+                ext = file_name.split(".")[-1]
+                size_filetypes[ext] = size_filetypes.get(ext, 0) + file_details["size"]
+
+        # Get user file sizes
+        size_user_content = site.content_manager.contents.execute(
+            "SELECT SUM(size) + SUM(size_files) AS size FROM content WHERE ?",
+            {"not__inner_path": contents}
+        ).fetchone()["size"]
+        if not size_user_content:
+            size_user_content = 0
+        size_filetypes["User data"] = size_user_content
+        size_total += size_user_content
+
+        # The missing difference is content.json sizes
+        if "json" in size_filetypes:
+            size_filetypes["json"] += max(0, site.settings["size"] - size_total)
+        size_total = size_other = site.settings["size"]
+
+        # Bar
+        for extension, color in extensions:
+            if extension == "Total":
+                continue
+            if extension == "Other":
+                size = max(0, size_other)
+            elif extension == "Image":
+                size = size_filetypes.get("jpg", 0) + size_filetypes.get("png", 0) + size_filetypes.get("gif", 0)
+                size_other -= size
+            else:
+                size = size_filetypes.get(extension, 0)
+                size_other -= size
+            if size_total == 0:
+                percent = 0
+            else:
+                percent = 100 * (float(size) / size_total)
+            percent = math.floor(percent * 100) / 100  # Floor to 2 digits
+            body.append(
+                """<li style='width: %.2f%%' class='%s back-%s' title="%s"></li>""" %
+                (percent, _[extension], color, _[extension])
+            )
+
+        # Legend
+        body.append("</ul><ul class='graph-legend'>")
+        for extension, color in extensions:
+            if extension == "Other":
+                size = max(0, size_other)
+            elif extension == "Image":
+                size = size_filetypes.get("jpg", 0) + size_filetypes.get("png", 0) + size_filetypes.get("gif", 0)
+            elif extension == "Total":
+                size = size_total
+            else:
+                size = size_filetypes.get(extension, 0)
+
+            if extension == "js":
+                title = "javascript"
+            else:
+                title = extension
+
+            if size > 1024 * 1024 * 10:  # Format as mB is more than 10mB
+                size_formatted = "%.0fMB" % (size / 1024 / 1024)
+            else:
+                size_formatted = "%.0fkB" % (size / 1024)
+
+            body.append("<li class='color-%s'><span>%s:</span><b>%s</b></li>" % (color, _[title], size_formatted))
+
+        body.append("</ul></li>")
+
+    def sidebarRenderSizeLimit(self, body, site):
+        free_space = helper.getFreeSpace() / 1024 / 1024
+        size = float(site.settings["size"]) / 1024 / 1024
+        size_limit = site.getSizeLimit()
+        percent_used = size / size_limit
+
+        body.append(_("""
+            <li>
+             <label>{_[Size limit]} <small>({_[limit used]}: {percent_used:.0%}, {_[free space]}: {free_space:,.0f}MB)</small></label>
+             <input type='text' class='text text-num' value="{size_limit}" id='input-sitelimit'/><span class='text-post'>MB</span>
+             <a href='#Set' class='button' id='button-sitelimit'>{_[Set]}</a>
+            </li>
+        """))
+
+    def sidebarRenderOptionalFileStats(self, body, site):
+        size_total = float(site.settings["size_optional"])
+        size_downloaded = float(site.settings["optional_downloaded"])
+
+        if not size_total:
+            return False
+
+        percent_downloaded = size_downloaded / size_total
+
+        size_formatted_total = size_total / 1024 / 1024
+        size_formatted_downloaded = size_downloaded / 1024 / 1024
+
+        body.append(_("""
+            <li>
+             <label>{_[Optional files]}</label>
+             <ul class='graph'>
+              <li style='width: 100%' class='total back-black' title="{_[Total size]}"></li>
+              <li style='width: {percent_downloaded:.0%}' class='connected back-green' title='{_[Downloaded files]}'></li>
+             </ul>
+             <ul class='graph-legend'>
+              <li class='color-green'><span>{_[Downloaded]}:</span><b>{size_formatted_downloaded:.2f}MB</b></li>
+              <li class='color-black'><span>{_[Total]}:</span><b>{size_formatted_total:.2f}MB</b></li>
+             </ul>
+            </li>
+        """))
+
+        return True
+
+    def sidebarRenderOptionalFileSettings(self, body, site):
+        if self.site.settings.get("autodownloadoptional"):
+            checked = "checked='checked'"
+        else:
+            checked = ""
+
+        body.append(_("""
+            <li>
+             <label>{_[Download and help distribute all files]}</label>
+             <input type="checkbox" class="checkbox" id="checkbox-autodownloadoptional" {checked}/><div class="checkbox-skin"></div>
+        """))
+
+        autodownload_bigfile_size_limit = int(site.settings.get("autodownload_bigfile_size_limit", config.autodownload_bigfile_size_limit))
+        body.append(_("""
+            <div class='settings-autodownloadoptional'>
+             <label>{_[Auto download big file size limit]}</label>
+             <input type='text' class='text text-num' value="{autodownload_bigfile_size_limit}" id='input-autodownload_bigfile_size_limit'/><span class='text-post'>MB</span>
+             <a href='#Set' class='button' id='button-autodownload_bigfile_size_limit'>{_[Set]}</a>
+            </div>
+        """))
+        body.append("</li>")
+
+    def sidebarRenderBadFiles(self, body, site):
+        body.append(_("""
+            <li>
+             <label>{_[Needs to be updated]}:</label>
+             <ul class='filelist'>
+        """))
+
+        i = 0
+        for bad_file, tries in site.bad_files.items():
+            i += 1
+            body.append(_("""<li class='color-red' title="{bad_file_path} ({tries})">{bad_filename}</li>""", {
+                "bad_file_path": bad_file,
+                "bad_filename": helper.getFilename(bad_file),
+                "tries": _.pluralize(tries, "{} try", "{} tries")
+            }))
+            if i > 30:
+                break
+
+        if len(site.bad_files) > 30:
+            num_bad_files = len(site.bad_files) - 30
+            body.append(_("""<li class='color-red'>{_[+ {num_bad_files} more]}</li>""", nested=True))
+
+        body.append("""
+             </ul>
+            </li>
+        """)
+
+    def sidebarRenderDbOptions(self, body, site):
+        if site.storage.db:
+            inner_path = site.storage.getInnerPath(site.storage.db.db_path)
+            size = float(site.storage.getSize(inner_path)) / 1024
+            feeds = len(site.storage.db.schema.get("feeds", {}))
+        else:
+            inner_path = _["No database found"]
+            size = 0.0
+            feeds = 0
+
+        body.append(_("""
+            <li>
+             <label>{_[Database]} <small>({size:.2f}kB, {_[search feeds]}: {_[{feeds} query]})</small></label>
+             <div class='flex'>
+              <input type='text' class='text disabled' value="{inner_path}" disabled='disabled'/>
+              <a href='#Reload' id="button-dbreload" class='button'>{_[Reload]}</a>
+              <a href='#Rebuild' id="button-dbrebuild" class='button'>{_[Rebuild]}</a>
+             </div>
+            </li>
+        """, nested=True))
+
+    def sidebarRenderIdentity(self, body, site):
+        auth_address = self.user.getAuthAddress(self.site.address, create=False)
+        rules = self.site.content_manager.getRules("data/users/%s/content.json" % auth_address)
+        if rules and rules.get("max_size"):
+            quota = rules["max_size"] / 1024
+            try:
+                content = site.content_manager.contents["data/users/%s/content.json" % auth_address]
+                used = len(json.dumps(content)) + sum([file["size"] for file in list(content["files"].values())])
+            except:
+                used = 0
+            used = used / 1024
+        else:
+            quota = used = 0
+
+        body.append(_("""
+            <li>
+             <label>{_[Identity address]} <small>({_[limit used]}: {used:.2f}kB / {quota:.2f}kB)</small></label>
+             <div class='flex'>
+              <span class='input text disabled'>{auth_address}</span>
+              <a href='#Change' class='button' id='button-identity'>{_[Change]}</a>
+             </div>
+            </li>
+        """))
+
+    def sidebarRenderControls(self, body, site):
+        auth_address = self.user.getAuthAddress(self.site.address, create=False)
+        if self.site.settings["serving"]:
+            class_pause = ""
+            class_resume = "hidden"
+        else:
+            class_pause = "hidden"
+            class_resume = ""
+
+        body.append(_("""
+            <li>
+             <label>{_[Site control]}</label>
+             <a href='#Update' class='button noupdate' id='button-update'>{_[Update]}</a>
+             <a href='#Pause' class='button {class_pause}' id='button-pause'>{_[Pause]}</a>
+             <a href='#Resume' class='button {class_resume}' id='button-resume'>{_[Resume]}</a>
+             <a href='#Delete' class='button noupdate' id='button-delete'>{_[Delete]}</a>
+            </li>
+        """))
+
+        donate_key = site.content_manager.contents.get("content.json", {}).get("donate", True)
+        site_address = self.site.address
+        body.append(_("""
+            <li>
+             <label>{_[Site address]}</label><br>
+             <div class='flex'>
+              <span class='input text disabled'>{site_address}</span>
+        """))
+        if donate_key == False or donate_key == "":
+            pass
+        elif (type(donate_key) == str or type(donate_key) == str) and len(donate_key) > 0:
+            body.append(_("""
+             </div>
+            </li>
+            <li>
+             <label>{_[Donate]}</label><br>
+             <div class='flex'>
+             {donate_key}
+            """))
+        else:
+            body.append(_("""
+              <a href='bitcoin:{site_address}' class='button' id='button-donate'>{_[Donate]}</a>
+            """))
+        body.append(_("""
+             </div>
+            </li>
+        """))
+
+    def sidebarRenderOwnedCheckbox(self, body, site):
+        if self.site.settings["own"]:
+            checked = "checked='checked'"
+        else:
+            checked = ""
+
+        body.append(_("""
+            <h2 class='owned-title'>{_[This is my site]}</h2>
+            <input type="checkbox" class="checkbox" id="checkbox-owned" {checked}/><div class="checkbox-skin"></div>
+        """))
+
+    def sidebarRenderOwnSettings(self, body, site):
+        title = site.content_manager.contents.get("content.json", {}).get("title", "")
+        description = site.content_manager.contents.get("content.json", {}).get("description", "")
+
+        body.append(_("""
+            <li>
+             <label for='settings-title'>{_[Site title]}</label>
+             <input type='text' class='text' value="{title}" id='settings-title'/>
+            </li>
+
+            <li>
+             <label for='settings-description'>{_[Site description]}</label>
+             <input type='text' class='text' value="{description}" id='settings-description'/>
+            </li>
+
+            <li>
+             <a href='#Save' class='button' id='button-settings'>{_[Save site settings]}</a>
+            </li>
+        """))
+
+    def sidebarRenderContents(self, body, site):
+        has_privatekey = bool(self.user.getSiteData(site.address, create=False).get("privatekey"))
+        if has_privatekey:
+            tag_privatekey = _("{_[Private key saved.]} <a href='#Forgot+private+key' id='privatekey-forgot' class='link-right'>{_[Forgot]}</a>")
+        else:
+            tag_privatekey = _("<a href='#Add+private+key' id='privatekey-add' class='link-right'>{_[Add saved private key]}</a>")
+
+        body.append(_("""
+            <li>
+             <label>{_[Content publishing]} <small class='label-right'>{tag_privatekey}</small></label>
+        """.replace("{tag_privatekey}", tag_privatekey)))
+
+        # Choose content you want to sign
+        body.append(_("""
+             <div class='flex'>
+              <input type='text' class='text' value="content.json" id='input-contents'/>
+              <a href='#Sign-and-Publish' id='button-sign-publish' class='button'>{_[Sign and publish]}</a>
+              <a href='#Sign-or-Publish' id='menu-sign-publish'>\u22EE</a>
+             </div>
+        """))
+
+        contents = ["content.json"]
+        contents += list(site.content_manager.contents.get("content.json", {}).get("includes", {}).keys())
+        body.append(_("<div class='contents'>{_[Choose]}: "))
+        for content in contents:
+            body.append(_("<a href='{content}' class='contents-content'>{content}</a> "))
+        body.append("</div>")
+        body.append("</li>")
+
+    def actionSidebarGetHtmlTag(self, to):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        site = self.site
+
+        body = []
+
+        body.append("<div>")
+        body.append("<a href='#Close' class='close'>&times;</a>")
+        body.append("<h1>%s</h1>" % html.escape(site.content_manager.contents.get("content.json", {}).get("title", ""), True))
+
+        body.append("<div class='globe loading'></div>")
+
+        body.append("<ul class='fields'>")
+
+        self.sidebarRenderPeerStats(body, site)
+        self.sidebarRenderTransferStats(body, site)
+        self.sidebarRenderFileStats(body, site)
+        self.sidebarRenderSizeLimit(body, site)
+        has_optional = self.sidebarRenderOptionalFileStats(body, site)
+        if has_optional:
+            self.sidebarRenderOptionalFileSettings(body, site)
+        self.sidebarRenderDbOptions(body, site)
+        self.sidebarRenderIdentity(body, site)
+        self.sidebarRenderControls(body, site)
+        if site.bad_files:
+            self.sidebarRenderBadFiles(body, site)
+
+        self.sidebarRenderOwnedCheckbox(body, site)
+        body.append("<div class='settings-owned'>")
+        self.sidebarRenderOwnSettings(body, site)
+        self.sidebarRenderContents(body, site)
+        body.append("</div>")
+        body.append("</ul>")
+        body.append("</div>")
+
+        body.append("<div class='menu template'>")
+        body.append("<a href='#'' class='menu-item template'>Template</a>")
+        body.append("</div>")
+
+        self.response(to, "".join(body))
+
+    def downloadGeoLiteDb(self, db_path):
+        import gzip
+        import shutil
+        from util import helper
+
+        if config.offline:
+            return False
+
+        self.log.info("Downloading GeoLite2 City database...")
+        self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0])
+        db_urls = [
+            "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
+            "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
+        ]
+        for db_url in db_urls:
+            downloadl_err = None
+            try:
+                # Download
+                response = helper.httpRequest(db_url)
+                data_size = response.getheader('content-length')
+                data_recv = 0
+                data = io.BytesIO()
+                while True:
+                    buff = response.read(1024 * 512)
+                    if not buff:
+                        break
+                    data.write(buff)
+                    data_recv += 1024 * 512
+                    if data_size:
+                        progress = int(float(data_recv) / int(data_size) * 100)
+                        self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress])
+                self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell())
+                data.seek(0)
+
+                # Unpack
+                with gzip.GzipFile(fileobj=data) as gzip_file:
+                    shutil.copyfileobj(gzip_file, open(db_path, "wb"))
+
+                self.cmd("progress", ["geolite-info", _["GeoLite2 City database downloaded!"], 100])
+                time.sleep(2)  # Wait for notify animation
+                self.log.info("GeoLite2 City database is ready at: %s" % db_path)
+                return True
+            except Exception as err:
+                download_err = err
+                self.log.error("Error downloading %s: %s" % (db_url, err))
+                pass
+        self.cmd("progress", [
+            "geolite-info",
+            _["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"].format(download_err, db_urls[0]),
+            -100
+        ])
+
+    def getLoc(self, geodb, ip):
+        global loc_cache
+
+        if ip in loc_cache:
+            return loc_cache[ip]
+        else:
+            try:
+                loc_data = geodb.get(ip)
+            except:
+                loc_data = None
+
+            if not loc_data or "location" not in loc_data:
+                loc_cache[ip] = None
+                return None
+
+            loc = {
+                "lat": loc_data["location"]["latitude"],
+                "lon": loc_data["location"]["longitude"],
+            }
+            if "city" in loc_data:
+                loc["city"] = loc_data["city"]["names"]["en"]
+
+            if "country" in loc_data:
+                loc["country"] = loc_data["country"]["names"]["en"]
+
+            loc_cache[ip] = loc
+            return loc
+
+    def getGeoipDb(self):
+        db_name = 'GeoLite2-City.mmdb'
+
+        sys_db_paths = []
+        if sys.platform == "linux":
+            sys_db_paths += ['/usr/share/GeoIP/' + db_name]
+
+        data_dir_db_path = os.path.join(config.data_dir, db_name)
+
+        db_paths = sys_db_paths + [data_dir_db_path]
+
+        for path in db_paths:
+            if os.path.isfile(path) and os.path.getsize(path) > 0:
+                return path
+
+        self.log.info("GeoIP database not found at [%s]. Downloading to: %s",
+                " ".join(db_paths), data_dir_db_path)
+        if self.downloadGeoLiteDb(data_dir_db_path):
+            return data_dir_db_path
+        return None
+
+    def getPeerLocations(self, peers):
+        import maxminddb
+
+        db_path = self.getGeoipDb()
+        if not db_path:
+            self.log.debug("Not showing peer locations: no GeoIP database")
+            return False
+
+        self.log.info("Loading GeoIP database from: %s" % db_path)
+        geodb = maxminddb.open_database(db_path)
+
+        peers = list(peers.values())
+        # Place bars
+        peer_locations = []
+        placed = {}  # Already placed bars here
+        for peer in peers:
+            # Height of bar
+            if peer.connection and peer.connection.last_ping_delay:
+                ping = round(peer.connection.last_ping_delay * 1000)
+            else:
+                ping = None
+            loc = self.getLoc(geodb, peer.ip)
+
+            if not loc:
+                continue
+            # Create position array
+            lat, lon = loc["lat"], loc["lon"]
+            latlon = "%s,%s" % (lat, lon)
+            if latlon in placed and helper.getIpType(peer.ip) == "ipv4":  # Dont place more than 1 bar to same place, fake repos using ip address last two part
+                lat += float(128 - int(peer.ip.split(".")[-2])) / 50
+                lon += float(128 - int(peer.ip.split(".")[-1])) / 50
+                latlon = "%s,%s" % (lat, lon)
+            placed[latlon] = True
+            peer_location = {}
+            peer_location.update(loc)
+            peer_location["lat"] = lat
+            peer_location["lon"] = lon
+            peer_location["ping"] = ping
+
+            peer_locations.append(peer_location)
+
+        # Append myself
+        for ip in self.site.connection_server.ip_external_list:
+            my_loc = self.getLoc(geodb, ip)
+            if my_loc:
+                my_loc["ping"] = 0
+                peer_locations.append(my_loc)
+
+        return peer_locations
+
+
+    def actionSidebarGetPeers(self, to):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+        try:
+            peer_locations = self.getPeerLocations(self.site.peers)
+            globe_data = []
+            ping_times = [
+                peer_location["ping"]
+                for peer_location in peer_locations
+                if peer_location["ping"]
+            ]
+            if ping_times:
+                ping_avg = sum(ping_times) / float(len(ping_times))
+            else:
+                ping_avg = 0
+
+            for peer_location in peer_locations:
+                if peer_location["ping"] == 0:  # Me
+                    height = -0.135
+                elif peer_location["ping"]:
+                    height = min(0.20, math.log(1 + peer_location["ping"] / ping_avg, 300))
+                else:
+                    height = -0.03
+
+                globe_data += [peer_location["lat"], peer_location["lon"], height]
+
+            self.response(to, globe_data)
+        except Exception as err:
+            self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err))
+            self.response(to, {"error": str(err)})
+
+    def actionSiteSetOwned(self, to, owned):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        if self.site.address == config.updatesite:
+            return self.response(to, "You can't change the ownership of the updater site")
+
+        self.site.settings["own"] = bool(owned)
+        self.site.updateWebsocket(owned=owned)
+
+    def actionUserSetSitePrivatekey(self, to, privatekey):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        site_data = self.user.sites[self.site.address]
+        site_data["privatekey"] = privatekey
+        self.site.updateWebsocket(set_privatekey=bool(privatekey))
+
+        return "ok"
+
+    def actionSiteSetAutodownloadoptional(self, to, owned):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        self.site.settings["autodownloadoptional"] = bool(owned)
+        self.site.bad_files = {}
+        gevent.spawn(self.site.update, check_files=True)
+        self.site.worker_manager.removeSolvedFileTasks()
+
+    def actionDbReload(self, to):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        self.site.storage.closeDb()
+        self.site.storage.getDb()
+
+        return self.response(to, "ok")
+
+    def actionDbRebuild(self, to):
+        permissions = self.getPermissions(to)
+        if "ADMIN" not in permissions:
+            return self.response(to, "You don't have permission to run this command")
+
+        result = self.site.storage.rebuildDb()
+
+        if not result:
+            return self.response(to, {"error": "Failed to rebuild database"})
+
+        return self.response(to, "ok")

plugins/Sidebar/ZipStream.py

@@ -0,0 +1,59 @@
+import io
+import os
+import zipfile
+
+class ZipStream(object):
+    def __init__(self, dir_path):
+        self.dir_path = dir_path
+        self.pos = 0
+        self.buff_pos = 0
+        self.zf = zipfile.ZipFile(self, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
+        self.buff = io.BytesIO()
+        self.file_list = self.getFileList()
+
+    def getFileList(self):
+        for root, dirs, files in os.walk(self.dir_path):
+            for file in files:
+                file_path = root + "/" + file
+                relative_path = os.path.join(os.path.relpath(root, self.dir_path), file)
+                yield file_path, relative_path
+        self.zf.close()
+
+    def read(self, size=60 * 1024):
+        for file_path, relative_path in self.file_list:
+            self.zf.write(file_path, relative_path)
+            if self.buff.tell() >= size:
+                break
+        self.buff.seek(0)
+        back = self.buff.read()
+        self.buff.truncate(0)
+        self.buff.seek(0)
+        self.buff_pos += len(back)
+        return back
+
+    def write(self, data):
+        self.pos += len(data)
+        self.buff.write(data)
+
+    def tell(self):
+        return self.pos
+
+    def seek(self, pos, whence=0):
+        if pos >= self.buff_pos:
+            self.buff.seek(pos - self.buff_pos, whence)
+            self.pos = pos
+
+    def flush(self):
+        pass
+
+
+if __name__ == "__main__":
+    zs = ZipStream(".")
+    out = open("out.zip", "wb")
+    while 1:
+        data = zs.read()
+        print("Write %s" % len(data))
+        if not data:
+            break
+        out.write(data)
+    out.close()

plugins/Sidebar/__init__.py

@@ -0,0 +1 @@
+from . import SidebarPlugin

plugins/Sidebar/languages/da.json

@@ -0,0 +1,81 @@
+{
+	"Peers": "Klienter",
+	"Connected": "Forbundet",
+	"Connectable": "Mulige",
+	"Connectable peers": "Mulige klienter",
+
+	"Data transfer": "Data overførsel",
+	"Received": "Modtaget",
+	"Received bytes": "Bytes modtaget",
+	"Sent": "Sendt",
+	"Sent bytes": "Bytes sendt",
+
+	"Files": "Filer",
+	"Total": "I alt",
+	"Image": "Image",
+	"Other": "Andet",
+	"User data": "Bruger data",
+
+	"Size limit": "Side max størrelse",
+	"limit used": "brugt",
+	"free space": "fri",
+	"Set": "Opdater",
+
+	"Optional files": "Valgfri filer",
+	"Downloaded": "Downloadet",
+	"Download and help distribute all files": "Download og hjælp med at dele filer",
+	"Total size": "Størrelse i alt",
+	"Downloaded files": "Filer downloadet",
+
+	"Database": "Database",
+	"search feeds": "søgninger",
+	"{feeds} query": "{feeds} søgninger",
+	"Reload": "Genindlæs",
+	"Rebuild": "Genopbyg",
+	"No database found": "Ingen database fundet",
+
+	"Identity address": "Autorisations ID",
+	"Change": "Skift",
+
+	"Update": "Opdater",
+	"Pause": "Pause",
+	"Resume": "Aktiv",
+	"Delete": "Slet",
+	"Are you sure?": "Er du sikker?",
+
+	"Site address": "Side addresse",
+	"Donate": "Doner penge",
+
+	"Missing files": "Manglende filer",
+	"{} try": "{} forsøg",
+	"{} tries": "{} forsøg",
+	"+ {num_bad_files} more": "+ {num_bad_files} mere",
+
+	"This is my site": "Dette er min side",
+	"Site title": "Side navn",
+	"Site description": "Side beskrivelse",
+	"Save site settings": "Gem side opsætning",
+
+	"Content publishing": "Indhold offentliggøres",
+	"Choose": "Vælg",
+	"Sign": "Signer",
+	"Publish": "Offentliggør",
+
+	"This function is disabled on this proxy": "Denne funktion er slået fra på denne ZeroNet proxyEz a funkció ki van kapcsolva ezen a proxy-n",
+	"GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}": "GeoLite2 City database kunne ikke downloades: {}!<br>Download venligst databasen manuelt og udpak i data folder:<br>{}",
+	"Downloading GeoLite2 City database (one time only, ~20MB)...": "GeoLite2 város adatbázis letöltése (csak egyszer kell, kb 20MB)...",
+	"GeoLite2 City database downloaded!": "GeoLite2 City database downloadet!",
+
+	"Are you sure?": "Er du sikker?",
+	"Site storage limit modified!": "Side max størrelse ændret!",
+	"Database schema reloaded!": "Database definition genindlæst!",
+	"Database rebuilding....": "Genopbygger database...",
+	"Database rebuilt!": "Database genopbygget!",
+	"Site updated!": "Side opdateret!",
+	"Delete this site": "Slet denne side",
+	"File write error: ": "Fejl ved skrivning af fil: ",
+	"Site settings saved!": "Side opsætning gemt!",
+	"Enter your private key:": "Indtast din private nøgle:",
+	" Signed!": " Signeret!",
+	"WebGL not supported": "WebGL er ikke supporteret"
+}

plugins/Sidebar/languages/de.json

@@ -0,0 +1,81 @@
+{
+	"Peers": "Peers",
+	"Connected": "Verbunden",
+	"Connectable": "Verbindbar",
+	"Connectable peers": "Verbindbare Peers",
+
+	"Data transfer": "Datei Transfer",
+	"Received": "Empfangen",
+	"Received bytes": "Empfangene Bytes",
+	"Sent": "Gesendet",
+	"Sent bytes": "Gesendete Bytes",
+
+	"Files": "Dateien",
+	"Total": "Gesamt",
+	"Image": "Bilder",
+	"Other": "Sonstiges",
+	"User data": "Nutzer Daten",
+
+	"Size limit": "Speicher Limit",
+	"limit used": "Limit benutzt",
+	"free space": "freier Speicher",
+	"Set": "Setzten",
+
+	"Optional files": "Optionale Dateien",
+	"Downloaded": "Heruntergeladen",
+	"Download and help distribute all files": "Herunterladen und helfen alle Dateien zu verteilen",
+	"Total size": "Gesamte Größe",
+	"Downloaded files": "Heruntergeladene Dateien",
+
+	"Database": "Datenbank",
+	"search feeds": "Feeds durchsuchen",
+	"{feeds} query": "{feeds} Abfrage",
+	"Reload": "Neu laden",
+	"Rebuild": "Neu bauen",
+	"No database found": "Keine Datenbank gefunden",
+
+	"Identity address": "Identitäts Adresse",
+	"Change": "Ändern",
+
+	"Update": "Aktualisieren",
+	"Pause": "Pausieren",
+	"Resume": "Fortsetzen",
+	"Delete": "Löschen",
+	"Are you sure?": "Bist du sicher?",
+
+	"Site address": "Seiten Adresse",
+	"Donate": "Spenden",
+
+	"Missing files": "Fehlende Dateien",
+	"{} try": "{} versuch",
+	"{} tries": "{} versuche",
+	"+ {num_bad_files} more": "+ {num_bad_files} mehr",
+
+	"This is my site": "Das ist meine Seite",
+	"Site title": "Seiten Titel",
+	"Site description": "Seiten Beschreibung",
+	"Save site settings": "Einstellungen der Seite speichern",
+
+	"Content publishing": "Inhaltsveröffentlichung",
+	"Choose": "Wähle",
+	"Sign": "Signieren",
+	"Publish": "Veröffentlichen",
+
+	"This function is disabled on this proxy": "Diese Funktion ist auf dieser Proxy deaktiviert",
+	"GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}": "GeoLite2 City Datenbank Download Fehler: {}!<br>Bitte manuell herunterladen und die Datei in das Datei Verzeichnis extrahieren:<br>{}",
+	"Downloading GeoLite2 City database (one time only, ~20MB)...": "Herunterladen der GeoLite2 City Datenbank (einmalig, ~20MB)...",
+	"GeoLite2 City database downloaded!": "GeoLite2 City Datenbank heruntergeladen!",
+
+	"Are you sure?": "Bist du sicher?",
+	"Site storage limit modified!": "Speicher Limit der Seite modifiziert!",
+	"Database schema reloaded!": "Datebank Schema neu geladen!",
+	"Database rebuilding....": "Datenbank neu bauen...",
+	"Database rebuilt!": "Datenbank neu gebaut!",
+	"Site updated!": "Seite aktualisiert!",
+	"Delete this site": "Diese Seite löschen",
+	"File write error: ": "Datei schreib fehler:",
+	"Site settings saved!": "Seiten Einstellungen gespeichert!",
+	"Enter your private key:": "Gib deinen privaten Schlüssel ein:",
+	" Signed!": " Signiert!",
+	"WebGL not supported": "WebGL nicht unterstützt"
+}

plugins/Sidebar/languages/es.json

@@ -0,0 +1,79 @@
+{
+	"Peers": "Pares",
+	"Connected": "Conectados",
+	"Connectable": "Conectables",
+	"Connectable peers": "Pares conectables",
+
+	"Data transfer": "Transferencia de datos",
+	"Received": "Recibidos",
+	"Received bytes": "Bytes recibidos",
+	"Sent": "Enviados",
+	"Sent bytes": "Bytes envidados",
+
+	"Files": "Ficheros",
+	"Total": "Total",
+	"Image": "Imagen",
+	"Other": "Otro",
+	"User data": "Datos del usuario",
+
+	"Size limit": "Límite de tamaño",
+	"limit used": "Límite utilizado",
+	"free space": "Espacio libre",
+	"Set": "Establecer",
+
+	"Optional files": "Ficheros opcionales",
+	"Downloaded": "Descargado",
+	"Download and help distribute all files": "Descargar y ayudar a distribuir todos los ficheros",
+	"Total size": "Tamaño total",
+	"Downloaded files": "Ficheros descargados",
+
+	"Database": "Base de datos",
+	"search feeds": "Fuentes de búsqueda",
+	"{feeds} query": "{feeds} consulta",
+	"Reload": "Recargar",
+	"Rebuild": "Reconstruir",
+	"No database found": "No se ha encontrado la base de datos",
+
+	"Identity address": "Dirección de la identidad",
+	"Change": "Cambiar",
+
+	"Update": "Actualizar",
+	"Pause": "Pausar",
+	"Resume": "Reanudar",
+	"Delete": "Borrar",
+
+	"Site address": "Dirección del sitio",
+	"Donate": "Donar",
+
+	"Missing files": "Ficheros perdidos",
+	"{} try": "{} intento",
+	"{} tries": "{} intentos",
+	"+ {num_bad_files} more": "+ {num_bad_files} más",
+
+	"This is my site": "Este es mi sitio",
+	"Site title": "Título del sitio",
+	"Site description": "Descripción del sitio",
+	"Save site settings": "Guardar la configuración del sitio",
+
+	"Content publishing": "Publicación del contenido",
+	"Choose": "Elegir",
+	"Sign": "Firmar",
+	"Publish": "Publicar",
+	"This function is disabled on this proxy": "Esta función está deshabilitada en este proxy",
+	"GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}": "¡Error de la base de datos GeoLite2： {}!<br>Por favor, descárgalo manualmente y descomprime al directorio de datos：<br>{}",
+	"Downloading GeoLite2 City database (one time only, ~20MB)...": "Descargando la base de datos de GeoLite2 (una única vez, ~20MB）...",
+	"GeoLite2 City database downloaded!": "¡Base de datos de GeoLite2 descargada!",
+
+	"Are you sure?": "¿Estás seguro?",
+	"Site storage limit modified!": "¡Límite de almacenamiento del sitio modificado!",
+	"Database schema reloaded!": "¡Esquema de la base de datos recargado!",
+	"Database rebuilding....": "Reconstruyendo la base de datos...",
+	"Database rebuilt!": "¡Base de datos reconstruida!",
+	"Site updated!": "¡Sitio actualizado!",
+	"Delete this site": "Borrar este sitio",
+	"File write error: ": "Error de escritura de fichero：",
+	"Site settings saved!": "¡Configuración del sitio guardada!",
+	"Enter your private key:": "Introduce tu clave privada：",
+	" Signed!": " ¡firmado!",
+	"WebGL not supported": "WebGL no está soportado"
+}

plugins/Sidebar/languages/fr.json

@@ -0,0 +1,82 @@
+{
+	"Peers": "Pairs",
+	"Connected": "Connectés",
+	"Connectable": "Accessibles",
+	"Connectable peers": "Pairs accessibles",
+
+	"Data transfer": "Données transférées",
+	"Received": "Reçues",
+	"Received bytes": "Bytes reçus",
+	"Sent": "Envoyées",
+	"Sent bytes": "Bytes envoyés",
+
+	"Files": "Fichiers",
+	"Total": "Total",
+	"Image": "Image",
+	"Other": "Autre",
+	"User data": "Utilisateurs",
+
+	"Size limit": "Taille maximale",
+	"limit used": "utlisé",
+	"free space": "libre",
+	"Set": "Modifier",
+
+	"Optional files": "Fichiers optionnels",
+	"Downloaded": "Téléchargé",
+	"Download and help distribute all files": "Télécharger et distribuer tous les fichiers",
+	"Total size": "Taille totale",
+	"Downloaded files": "Fichiers téléchargés",
+
+	"Database": "Base de données",
+	"search feeds": "recherche",
+	"{feeds} query": "{feeds} requête",
+	"Reload": "Recharger",
+	"Rebuild": "Reconstruire",
+	"No database found": "Aucune base de données trouvée",
+
+	"Identity address": "Adresse d'identité",
+	"Change": "Modifier",
+
+	"Site control": "Opérations",
+	"Update": "Mettre à jour",
+	"Pause": "Suspendre",
+	"Resume": "Reprendre",
+	"Delete": "Supprimer",
+	"Are you sure?": "Êtes-vous certain?",
+
+	"Site address": "Adresse du site",
+	"Donate": "Faire un don",
+
+	"Missing files": "Fichiers manquants",
+	"{} try": "{} essai",
+	"{} tries": "{} essais",
+	"+ {num_bad_files} more": "+ {num_bad_files} manquants",
+
+	"This is my site": "Ce site m'appartient",
+	"Site title": "Nom du site",
+	"Site description": "Description du site",
+	"Save site settings": "Enregistrer les paramètres",
+
+	"Content publishing": "Publication du contenu",
+	"Choose": "Sélectionner",
+	"Sign": "Signer",
+	"Publish": "Publier",
+
+	"This function is disabled on this proxy": "Cette fonction est désactivé sur ce proxy",
+	"GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}": "Erreur au téléchargement de la base de données GeoLite2: {}!<br>Téléchargez et décompressez dans le dossier data:<br>{}",
+	"Downloading GeoLite2 City database (one time only, ~20MB)...": "Téléchargement de la base de données GeoLite2 (une seule fois, ~20MB)...",
+	"GeoLite2 City database downloaded!": "Base de données GeoLite2 téléchargée!",
+
+	"Are you sure?": "Êtes-vous certain?",
+	"Site storage limit modified!": "Taille maximale modifiée!",
+	"Database schema reloaded!": "Base de données rechargée!",
+	"Database rebuilding....": "Reconstruction de la base de données...",
+	"Database rebuilt!": "Base de données reconstruite!",
+	"Site updated!": "Site mis à jour!",
+	"Delete this site": "Supprimer ce site",
+	"File write error: ": "Erreur à l'écriture du fichier: ",
+	"Site settings saved!": "Paramètres du site enregistrés!",
+	"Enter your private key:": "Entrez votre clé privée:",
+	" Signed!": " Signé!",
+	"WebGL not supported": "WebGL n'est pas supporté"
+}

plugins/Sidebar/languages/hu.json

@@ -0,0 +1,82 @@
+{
+	"Peers": "Csatlakozási pontok",
+	"Connected": "Csaltakozva",
+	"Connectable": "Csatlakozható",
+	"Connectable peers": "Csatlakozható peer-ek",
+
+	"Data transfer": "Adatátvitel",
+	"Received": "Fogadott",
+	"Received bytes": "Fogadott byte-ok",
+	"Sent": "Küldött",
+	"Sent bytes": "Küldött byte-ok",
+
+	"Files": "Fájlok",
+	"Total": "Összesen",
+	"Image": "Kép",
+	"Other": "Egyéb",
+	"User data": "Felh. adat",
+
+	"Size limit": "Méret korlát",
+	"limit used": "felhasznált",
+	"free space": "szabad hely",
+	"Set": "Beállít",
+
+	"Optional files": "Opcionális fájlok",
+	"Downloaded": "Letöltött",
+	"Download and help distribute all files": "Minden opcionális fájl letöltése",
+	"Total size": "Teljes méret",
+	"Downloaded files": "Letöltve",
+
+	"Database": "Adatbázis",
+	"search feeds": "Keresés források",
+	"{feeds} query": "{feeds} lekérdezés",
+	"Reload": "Újratöltés",
+	"Rebuild": "Újraépítés",
+	"No database found": "Adatbázis nem található",
+
+	"Identity address": "Azonosító cím",
+	"Change": "Módosít",
+
+	"Site control": "Oldal műveletek",
+	"Update": "Frissít",
+	"Pause": "Szünteltet",
+	"Resume": "Folytat",
+	"Delete": "Töröl",
+	"Are you sure?": "Biztos vagy benne?",
+
+	"Site address": "Oldal címe",
+	"Donate": "Támogatás",
+
+	"Missing files": "Hiányzó fájlok",
+	"{} try": "{} próbálkozás",
+	"{} tries": "{} próbálkozás",
+	"+ {num_bad_files} more": "+ még {num_bad_files} darab",
+
+	"This is my site": "Ez az én oldalam",
+	"Site title": "Oldal neve",
+	"Site description": "Oldal leírása",
+	"Save site settings": "Oldal beállítások mentése",
+
+	"Content publishing": "Tartalom publikálás",
+	"Choose": "Válassz",
+	"Sign": "Aláírás",
+	"Publish": "Publikálás",
+
+	"This function is disabled on this proxy": "Ez a funkció ki van kapcsolva ezen a proxy-n",
+	"GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}": "GeoLite2 város adatbázis letöltési hiba: {}!<br>A térképhez töltsd le és csomagold ki a data könyvtárba:<br>{}",
+	"Downloading GeoLite2 City database (one time only, ~20MB)...": "GeoLite2 város adatbázis letöltése (csak egyszer kell, kb 20MB)...",
+	"GeoLite2 City database downloaded!": "GeoLite2 város adatbázis letöltve!",
+
+	"Are you sure?": "Biztos vagy benne?",
+	"Site storage limit modified!": "Az oldalt méret korlát módosítva!",
+	"Database schema reloaded!": "Adatbázis séma újratöltve!",
+	"Database rebuilding....": "Adatbázis újraépítés...",
+	"Database rebuilt!": "Adatbázis újraépítve!",
+	"Site updated!": "Az oldal frissítve!",
+	"Delete this site": "Az oldal törlése",
+	"File write error: ": "Fájl írási hiba: ",
+	"Site settings saved!": "Az oldal beállításai elmentve!",
+	"Enter your private key:": "Add meg a prviát kulcsod:",
+	" Signed!": " Aláírva!",
+	"WebGL not supported": "WebGL nem támogatott"
+}