diff --git a/.forgejo/workflows/build-on-commit.yml b/.forgejo/workflows/build-on-commit.yml
deleted file mode 100644
index e8f0d2e3..00000000
--- a/.forgejo/workflows/build-on-commit.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-name: Build Docker Image on Commit
-
-on:
- push:
- branches:
- - main
- tags:
- - '!' # Exclude tags
-
-jobs:
- build-and-publish:
- runs-on: docker-builder
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Set REPO_VARS
- id: repo-url
- run: |
- echo "REPO_HOST=$(echo "${{ github.server_url }}" | sed 's~http[s]*://~~g')" >> $GITHUB_ENV
- echo "REPO_PATH=${{ github.repository }}" >> $GITHUB_ENV
-
- - name: Login to OCI registry
- run: |
- echo "${{ secrets.OCI_TOKEN }}" | docker login $REPO_HOST -u "${{ secrets.OCI_USER }}" --password-stdin
-
- - name: Build and push Docker images
- run: |
- # Build Docker image with commit SHA
- docker build -t $REPO_HOST/$REPO_PATH:${{ github.sha }} .
- docker push $REPO_HOST/$REPO_PATH:${{ github.sha }}
-
- # Build Docker image with nightly tag
- docker tag $REPO_HOST/$REPO_PATH:${{ github.sha }} $REPO_HOST/$REPO_PATH:nightly
- docker push $REPO_HOST/$REPO_PATH:nightly
-
- # Remove local images to save storage
- docker rmi $REPO_HOST/$REPO_PATH:${{ github.sha }}
- docker rmi $REPO_HOST/$REPO_PATH:nightly
diff --git a/.forgejo/workflows/build-on-tag.yml b/.forgejo/workflows/build-on-tag.yml
deleted file mode 100644
index 888102b6..00000000
--- a/.forgejo/workflows/build-on-tag.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-name: Build and Publish Docker Image on Tag
-
-on:
- push:
- tags:
- - '*'
-
-jobs:
- build-and-publish:
- runs-on: docker-builder
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Set REPO_VARS
- id: repo-url
- run: |
- echo "REPO_HOST=$(echo "${{ github.server_url }}" | sed 's~http[s]*://~~g')" >> $GITHUB_ENV
- echo "REPO_PATH=${{ github.repository }}" >> $GITHUB_ENV
-
- - name: Login to OCI registry
- run: |
- echo "${{ secrets.OCI_TOKEN }}" | docker login $REPO_HOST -u "${{ secrets.OCI_USER }}" --password-stdin
-
- - name: Build and push Docker image
- run: |
- TAG=${{ github.ref_name }} # Get the tag name from the context
- # Build and push multi-platform Docker images
- docker build -t $REPO_HOST/$REPO_PATH:$TAG --push .
- # Tag and push latest
- docker tag $REPO_HOST/$REPO_PATH:$TAG $REPO_HOST/$REPO_PATH:latest
- docker push $REPO_HOST/$REPO_PATH:latest
-
- # Remove the local image to save storage
- docker rmi $REPO_HOST/$REPO_PATH:$TAG
- docker rmi $REPO_HOST/$REPO_PATH:latest
\ No newline at end of file
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
index aab991d5..8c9f6621 100644
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,10 +1 @@
-github: canewsin
-patreon: # Replace with a single Patreon username e.g., user1
-open_collective: # Replace with a single Open Collective username e.g., user1
-ko_fi: canewsin
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: canewsin
-issuehunt: # Replace with a single IssueHunt username e.g., user1
-otechie: # Replace with a single Otechie username e.g., user1
-custom: ['https://paypal.me/PramUkesh', 'https://zerolink.ml/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/']
+custom: https://zerolink.ml/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
deleted file mode 100644
index 27b5c924..00000000
--- a/.github/workflows/codeql-analysis.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
- push:
- branches: [ py3-latest ]
- pull_request:
- # The branches below must be a subset of the branches above
- branches: [ py3-latest ]
- schedule:
- - cron: '32 19 * * 2'
-
-jobs:
- analyze:
- name: Analyze
- runs-on: ubuntu-latest
- permissions:
- actions: read
- contents: read
- security-events: write
-
- strategy:
- fail-fast: false
- matrix:
- language: [ 'javascript', 'python' ]
- # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
- # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
-
- # Initializes the CodeQL tools for scanning.
- - name: Initialize CodeQL
- uses: github/codeql-action/init@v2
- with:
- languages: ${{ matrix.language }}
- # If you wish to specify custom queries, you can do so here or in a config file.
- # By default, queries listed here will override any specified in a config file.
- # Prefix the list here with "+" to use these queries and those in the config file.
-
- # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
- # queries: security-extended,security-and-quality
-
-
- # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
- # If this step fails, then you should remove it and run the build manually (see below)
- - name: Autobuild
- uses: github/codeql-action/autobuild@v2
-
- # ℹ️ Command-line programs to run using the OS shell.
- # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
- # If the Autobuild fails above, remove it and uncomment the following three lines.
- # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
- # - run: |
- # echo "Run, Build Application using script"
- # ./location_of_script_within_repo/buildscript.sh
-
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2bdcaf95..6eaf3c6b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -4,48 +4,49 @@ on: [push, pull_request]
jobs:
test:
- runs-on: ubuntu-20.04
+
+ runs-on: ubuntu-18.04
strategy:
max-parallel: 16
matrix:
- python-version: ["3.7", "3.8", "3.9"]
+ python-version: [3.6, 3.7, 3.8, 3.9]
steps:
- - name: Checkout ZeroNet
- uses: actions/checkout@v2
- with:
- submodules: "true"
+ - name: Checkout ZeroNet
+ uses: actions/checkout@v2
+ with:
+ submodules: 'true'
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
- - name: Prepare for installation
- run: |
- python3 -m pip install setuptools
- python3 -m pip install --upgrade pip wheel
- python3 -m pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
+ - name: Prepare for installation
+ run: |
+ python3 -m pip install setuptools
+ python3 -m pip install --upgrade pip wheel
+ python3 -m pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
- - name: Install
- run: |
- python3 -m pip install --upgrade -r requirements.txt
- python3 -m pip list
+ - name: Install
+ run: |
+ python3 -m pip install --upgrade -r requirements.txt
+ python3 -m pip list
- - name: Prepare for tests
- run: |
- openssl version -a
- echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6
+ - name: Prepare for tests
+ run: |
+ openssl version -a
+ echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6
- - name: Test
- run: |
- catchsegv python3 -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini
- export ZERONET_LOG_DIR="log/CryptMessage"; catchsegv python3 -m pytest -x plugins/CryptMessage/Test
- export ZERONET_LOG_DIR="log/Bigfile"; catchsegv python3 -m pytest -x plugins/Bigfile/Test
- export ZERONET_LOG_DIR="log/AnnounceLocal"; catchsegv python3 -m pytest -x plugins/AnnounceLocal/Test
- export ZERONET_LOG_DIR="log/OptionalManager"; catchsegv python3 -m pytest -x plugins/OptionalManager/Test
- export ZERONET_LOG_DIR="log/Multiuser"; mv plugins/disabled-Multiuser plugins/Multiuser && catchsegv python -m pytest -x plugins/Multiuser/Test
- export ZERONET_LOG_DIR="log/Bootstrapper"; mv plugins/disabled-Bootstrapper plugins/Bootstrapper && catchsegv python -m pytest -x plugins/Bootstrapper/Test
- find src -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
- find plugins -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
- flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/
+ - name: Test
+ run: |
+ catchsegv python3 -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini
+ export ZERONET_LOG_DIR="log/CryptMessage"; catchsegv python3 -m pytest -x plugins/CryptMessage/Test
+ export ZERONET_LOG_DIR="log/Bigfile"; catchsegv python3 -m pytest -x plugins/Bigfile/Test
+ export ZERONET_LOG_DIR="log/AnnounceLocal"; catchsegv python3 -m pytest -x plugins/AnnounceLocal/Test
+ export ZERONET_LOG_DIR="log/OptionalManager"; catchsegv python3 -m pytest -x plugins/OptionalManager/Test
+ export ZERONET_LOG_DIR="log/Multiuser"; mv plugins/disabled-Multiuser plugins/Multiuser && catchsegv python -m pytest -x plugins/Multiuser/Test
+ export ZERONET_LOG_DIR="log/Bootstrapper"; mv plugins/disabled-Bootstrapper plugins/Bootstrapper && catchsegv python -m pytest -x plugins/Bootstrapper/Test
+ find src -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
+ find plugins -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')"
+ flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/
diff --git a/.gitignore b/.gitignore
index 636cd115..38dd3a34 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,7 +7,6 @@ __pycache__/
# Hidden files
.*
-!/.forgejo
!/.github
!/.gitignore
!/.travis.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6974d18a..b49b9ef6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,85 +1,6 @@
-### ZeroNet 0.9.0 (2023-07-12) Rev4630
- - Fix RDos Issue in Plugins https://github.com/ZeroNetX/ZeroNet-Plugins/pull/9
- - Add trackers to Config.py for failsafety incase missing trackers.txt
- - Added Proxy links
- - Fix pysha3 dep installation issue
- - FileRequest -> Remove Unnecessary check, Fix error wording
- - Fix Response when site is missing for `actionAs`
+### ZeroNet 0.7.2 (2020-09-?) Rev4206?
-### ZeroNet 0.8.5 (2023-02-12) Rev4625
- - Fix(https://github.com/ZeroNetX/ZeroNet/pull/202) for SSL cert gen failed on Windows.
- - default theme-class for missing value in `users.json`.
- - Fetch Stats Plugin changes.
-
-### ZeroNet 0.8.4 (2022-12-12) Rev4620
- - Increase Minimum Site size to 25MB.
-
-### ZeroNet 0.8.3 (2022-12-11) Rev4611
- - main.py -> Fix accessing unassigned varible
- - ContentManager -> Support for multiSig
- - SiteStrorage.py -> Fix accessing unassigned varible
- - ContentManager.py Improve Logging of Valid Signers
-
-### ZeroNet 0.8.2 (2022-11-01) Rev4610
- - Fix Startup Error when plugins dir missing
- - Move trackers to seperate file & Add more trackers
- - Config:: Skip loading missing tracker files
- - Added documentation for getRandomPort fn
-
-### ZeroNet 0.8.1 (2022-10-01) Rev4600
- - fix readdress loop (cherry-pick previously added commit from conservancy)
- - Remove Patreon badge
- - Update README-ru.md (#177)
- - Include inner_path of failed request for signing in error msg and response
- - Don't Fail Silently When Cert is Not Selected
- - Console Log Updates, Specify min supported ZeroNet version for Rust version Protocol Compatibility
- - Update FUNDING.yml
-
-### ZeroNet 0.8.0 (2022-05-27) Rev4591
- - Revert File Open to catch File Access Errors.
-
-### ZeroNet 0.7.9-patch (2022-05-26) Rev4586
- - Use xescape(s) from zeronet-conservancy
- - actionUpdate response Optimisation
- - Fetch Plugins Repo Updates
- - Fix Unhandled File Access Errors
- - Create codeql-analysis.yml
-
-### ZeroNet 0.7.9 (2022-05-26) Rev4585
- - Rust Version Compatibility for update Protocol msg
- - Removed Non Working Trakers.
- - Dynamically Load Trackers from Dashboard Site.
- - Tracker Supply Improvements.
- - Fix Repo Url for Bug Report
- - First Party Tracker Update Service using Dashboard Site.
- - remove old v2 onion service [#158](https://github.com/ZeroNetX/ZeroNet/pull/158)
-
-### ZeroNet 0.7.8 (2022-03-02) Rev4580
- - Update Plugins with some bug fixes and Improvements
-
-### ZeroNet 0.7.6 (2022-01-12) Rev4565
- - Sync Plugin Updates
- - Clean up tor v3 patch [#115](https://github.com/ZeroNetX/ZeroNet/pull/115)
- - Add More Default Plugins to Repo
- - Doubled Site Publish Limits
- - Update ZeroNet Repo Urls [#103](https://github.com/ZeroNetX/ZeroNet/pull/103)
- - UI/UX: Increases Size of Notifications Close Button [#106](https://github.com/ZeroNetX/ZeroNet/pull/106)
- - Moved Plugins to Seperate Repo
- - Added `access_key` variable in Config, this used to access restrited plugins when multiuser plugin is enabled. When MultiUserPlugin is enabled we cannot access some pages like /Stats, this key will remove such restriction with access key.
- - Added `last_connection_id_current_version` to ConnectionServer, helpful to estimate no of connection from current client version.
- - Added current version: connections to /Stats page. see the previous point.
-
-### ZeroNet 0.7.5 (2021-11-28) Rev4560
- - Add more default trackers
- - Change default homepage address to `1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`
- - Change default update site address to `1Update8crprmciJHwp2WXqkx2c4iYp18`
-
-### ZeroNet 0.7.3 (2021-11-28) Rev4555
- - Fix xrange is undefined error
- - Fix Incorrect viewport on mobile while loading
- - Tor-V3 Patch by anonymoose
-
### ZeroNet 0.7.1 (2019-07-01) Rev4206
### Added
diff --git a/README-ru.md b/README-ru.md
index 7d557727..1d0bafc1 100644
--- a/README-ru.md
+++ b/README-ru.md
@@ -3,131 +3,206 @@
[简体中文](./README-zh-cn.md)
[English](./README.md)
-Децентрализованные вебсайты, использующие криптографию Bitcoin и протокол BitTorrent — https://zeronet.dev ([Зеркало в ZeroNet](http://127.0.0.1:43110/1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX/)). В отличии от Bitcoin, ZeroNet'у не требуется блокчейн для работы, однако он использует ту же криптографию, чтобы обеспечить сохранность и проверку данных.
+Децентрализованные вебсайты использующие Bitcoin криптографию и BitTorrent сеть - https://zeronet.dev
+
## Зачем?
-- Мы верим в открытую, свободную, и неподдающуюся цензуре сеть и связь.
-- Нет единой точки отказа: Сайт остаётся онлайн, пока его обслуживает хотя бы 1 пир.
-- Нет затрат на хостинг: Сайты обслуживаются посетителями.
-- Невозможно отключить: Он нигде, потому что он везде.
-- Скорость и возможность работать без Интернета: Вы сможете получить доступ к сайту, потому что его копия хранится на вашем компьютере и у ваших пиров.
+* Мы верим в открытую, свободную, и не отцензуренную сеть и коммуникацию.
+* Нет единой точки отказа: Сайт онлайн пока по крайней мере 1 пир обслуживает его.
+* Никаких затрат на хостинг: Сайты обслуживаются посетителями.
+* Невозможно отключить: Он нигде, потому что он везде.
+* Быстр и работает оффлайн: Вы можете получить доступ к сайту, даже если Интернет недоступен.
+
## Особенности
+ * Обновляемые в реальном времени сайты
+ * Поддержка Namecoin .bit доменов
+ * Лёгок в установке: распаковал & запустил
+ * Клонирование вебсайтов в один клик
+ * Password-less [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
+ based authorization: Ваша учетная запись защищена той же криптографией, что и ваш Bitcoin-кошелек
+ * Встроенный SQL-сервер с синхронизацией данных P2P: Позволяет упростить разработку сайта и ускорить загрузку страницы
+ * Анонимность: Полная поддержка сети Tor с помощью скрытых служб .onion вместо адресов IPv4
+ * TLS зашифрованные связи
+ * Автоматическое открытие uPnP порта
+ * Плагин для поддержки многопользовательской (openproxy)
+ * Работает с любыми браузерами и операционными системами
-- Обновление сайтов в реальном времени
-- Поддержка доменов `.bit` ([Namecoin](https://www.namecoin.org))
-- Легкая установка: просто распакуйте и запустите
-- Клонирование сайтов "в один клик"
-- Беспарольная [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
- авторизация: Ваша учетная запись защищена той же криптографией, что и ваш Bitcoin-кошелек
-- Встроенный SQL-сервер с синхронизацией данных P2P: Позволяет упростить разработку сайта и ускорить загрузку страницы
-- Анонимность: Полная поддержка сети Tor, используя скрытые службы `.onion` вместо адресов IPv4
-- Зашифрованное TLS подключение
-- Автоматическое открытие UPnP–порта
-- Плагин для поддержки нескольких пользователей (openproxy)
-- Работа с любыми браузерами и операционными системами
-
-## Текущие ограничения
-
-- Файловые транзакции не сжаты
-- Нет приватных сайтов
## Как это работает?
-- После запуска `zeronet.py` вы сможете посещать сайты в ZeroNet, используя адрес
- `http://127.0.0.1:43110/{zeronet_адрес}`
- (Например: `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`).
-- Когда вы посещаете новый сайт в ZeroNet, он пытается найти пиров с помощью протокола BitTorrent,
- чтобы скачать у них файлы сайта (HTML, CSS, JS и т.д.).
-- После посещения сайта вы тоже становитесь его пиром.
-- Каждый сайт содержит файл `content.json`, который содержит SHA512 хеши всех остальные файлы
- и подпись, созданную с помощью закрытого ключа сайта.
-- Если владелец сайта (тот, кто владеет закрытым ключом для адреса сайта) изменяет сайт, он
+* После запуска `zeronet.py` вы сможете посетить зайты (zeronet сайты) используя адрес
+ `http://127.0.0.1:43110/{zeronet_address}`
+(например. `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`).
+* Когда вы посещаете новый сайт zeronet, он пытается найти пиров с помощью BitTorrent
+ чтобы загрузить файлы сайтов (html, css, js ...) из них.
+* Каждый посещенный зайт также обслуживается вами. (Т.е хранится у вас на компьютере)
+* Каждый сайт содержит файл `content.json`, который содержит все остальные файлы в хэше sha512
+ и подпись, созданную с использованием частного ключа сайта.
+* Если владелец сайта (у которого есть закрытый ключ для адреса сайта) изменяет сайт, то он/она
подписывает новый `content.json` и публикует его для пиров. После этого пиры проверяют целостность `content.json`
- (используя подпись), скачвают изменённые файлы и распространяют новый контент для других пиров.
+ (используя подпись), они загружают измененные файлы и публикуют новый контент для других пиров.
+
+#### [Слайд-шоу о криптографии ZeroNet, обновлениях сайтов, многопользовательских сайтах »](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000)
+#### [Часто задаваемые вопросы »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/)
+
+#### [Документация разработчика ZeroNet »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
-[Презентация о криптографии ZeroNet, обновлениях сайтов, многопользовательских сайтах »](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000)
-[Часто задаваемые вопросы »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/)
-[Документация разработчика ZeroNet »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
## Скриншоты


-[Больше скриншотов в документации ZeroNet »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/)
-## Как присоединиться?
+#### [Больше скриншотов в ZeroNet документации »](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/)
-### Windows
-- Скачайте и распакуйте архив [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26МБ)
-- Запустите `ZeroNet.exe`
+## Как вступить
-### macOS
+* Скачайте ZeroBundle пакет:
+ * [Microsoft Windows](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip)
+ * [Apple macOS](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip)
+ * [Linux 64-bit](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip)
+ * [Linux 32-bit](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip)
+* Распакуйте где угодно
+* Запустите `ZeroNet.exe` (win), `ZeroNet(.app)` (osx), `ZeroNet.sh` (linux)
-- Скачайте и распакуйте архив [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14МБ)
-- Запустите `ZeroNet.app`
+### Linux терминал
-### Linux (64 бит)
+* `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip`
+* `unzip ZeroNet-linux.zip`
+* `cd ZeroNet-linux`
+* Запустите с помощью `./ZeroNet.sh`
-- Скачайте и распакуйте архив [ZeroNet-linux.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip) (14МБ)
-- Запустите `./ZeroNet.sh`
+Он загружает последнюю версию ZeroNet, затем запускает её автоматически.
-> **Note**
-> Запустите таким образом: `./ZeroNet.sh --ui_ip '*' --ui_restrict ваш_ip_адрес`, чтобы разрешить удалённое подключение к веб–интерфейсу.
+#### Ручная установка для Debian Linux
-### Docker
+* `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip`
+* `unzip ZeroNet-src.zip`
+* `cd ZeroNet`
+* `sudo apt-get update`
+* `sudo apt-get install python3-pip`
+* `sudo python3 -m pip install -r requirements.txt`
+* Запустите с помощью `python3 zeronet.py`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
-Официальный образ находится здесь: https://hub.docker.com/r/canewsin/zeronet/
+### [Arch Linux](https://www.archlinux.org)
-### Android (arm, arm64, x86)
+* `git clone https://aur.archlinux.org/zeronet.git`
+* `cd zeronet`
+* `makepkg -srci`
+* `systemctl start zeronet`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
-- Для работы требуется Android как минимум версии 5.0 Lollipop
-- [
](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile)
-- Скачать APK: https://github.com/canewsin/zeronet_mobile/releases
+Смотрите [ArchWiki](https://wiki.archlinux.org)'s [ZeroNet
+article](https://wiki.archlinux.org/index.php/ZeroNet) для дальнейшей помощи.
-### Android (arm, arm64, x86) Облегчённый клиент только для просмотра (1МБ)
+### [Gentoo Linux](https://www.gentoo.org)
-- Для работы требуется Android как минимум версии 4.1 Jelly Bean
-- [
](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite)
+* [`layman -a raiagent`](https://github.com/leycec/raiagent)
+* `echo '>=net-vpn/zeronet-0.5.4' >> /etc/portage/package.accept_keywords`
+* *(Опционально)* Включить поддержку Tor: `echo 'net-vpn/zeronet tor' >>
+ /etc/portage/package.use`
+* `emerge zeronet`
+* `rc-service zeronet start`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
-### Установка из исходного кода
+Смотрите `/usr/share/doc/zeronet-*/README.gentoo.bz2` для дальнейшей помощи.
-```sh
-wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip
-unzip ZeroNet-src.zip
-cd ZeroNet
-sudo apt-get update
-sudo apt-get install python3-pip
-sudo python3 -m pip install -r requirements.txt
+### [FreeBSD](https://www.freebsd.org/)
+
+* `pkg install zeronet` or `cd /usr/ports/security/zeronet/ && make install clean`
+* `sysrc zeronet_enable="YES"`
+* `service zeronet start`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+### [Vagrant](https://www.vagrantup.com/)
+
+* `vagrant up`
+* Подключитесь к VM с помощью `vagrant ssh`
+* `cd /vagrant`
+* Запустите `python3 zeronet.py --ui_ip 0.0.0.0`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+### [Docker](https://www.docker.com/)
+* `docker run -d -v :/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 canewsin/zeronet`
+* Это изображение Docker включает в себя прокси-сервер Tor, который по умолчанию отключён.
+ Остерегайтесь что некоторые хостинг-провайдеры могут не позволить вам запускать Tor на своих серверах.
+ Если вы хотите включить его,установите переменную среды `ENABLE_TOR` в` true` (по умолчанию: `false`) Например:
+
+ `docker run -d -e "ENABLE_TOR=true" -v :/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 canewsin/zeronet`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+### [Virtualenv](https://virtualenv.readthedocs.org/en/latest/)
+
+* `virtualenv env`
+* `source env/bin/activate`
+* `pip install msgpack gevent`
+* `python3 zeronet.py`
+* Откройте http://127.0.0.1:43110/ в вашем браузере.
+
+## Текущие ограничения
+
+* Файловые транзакции не сжаты
+* Нет приватных сайтов
+
+
+## Как я могу создать сайт в Zeronet?
+
+Завершите работу zeronet, если он запущен
+
+```bash
+$ zeronet.py siteCreate
+...
+- Site private key (Приватный ключ сайта): 23DKQpzxhbVBrAtvLEc2uvk7DZweh4qL3fn3jpM3LgHDczMK2TtYUq
+- Site address (Адрес сайта): 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+- Site created! (Сайт создан)
+$ zeronet.py
+...
```
-- Запустите `python3 zeronet.py`
-Откройте приветственную страницу ZeroHello в вашем браузере по ссылке http://127.0.0.1:43110/
+Поздравляем, вы закончили! Теперь каждый может получить доступ к вашему зайту используя
+`http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2`
-## Как мне создать сайт в ZeroNet?
+Следующие шаги: [ZeroNet Developer Documentation](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
-- Кликните на **⋮** > **"Create new, empty site"** в меню на сайте [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d).
-- Вы будете **перенаправлены** на совершенно новый сайт, который может быть изменён только вами!
-- Вы можете найти и изменить контент вашего сайта в каталоге **data/[адрес_вашего_сайта]**
-- После изменений откройте ваш сайт, переключите влево кнопку "0" в правом верхнем углу, затем нажмите кнопки **sign** и **publish** внизу
-Следующие шаги: [Документация разработчика ZeroNet](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/)
+## Как я могу модифицировать Zeronet сайт?
+
+* Измените файлы расположенные в data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 директории.
+ Когда закончите с изменением:
+
+```bash
+$ zeronet.py siteSign 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+- Signing site (Подпись сайта): 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2...
+Private key (Приватный ключ) (input hidden):
+```
+
+* Введите секретный ключ, который вы получили при создании сайта, потом:
+
+```bash
+$ zeronet.py sitePublish 13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+...
+Site:13DNDk..bhC2 Publishing to 3/10 peers...
+Site:13DNDk..bhC2 Successfuly published to 3 peers
+- Serving files....
+```
+
+* Вот и всё! Вы успешно подписали и опубликовали свои изменения.
+
## Поддержите проект
-
-- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Рекомендуем)
+- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Preferred)
- LiberaPay: https://liberapay.com/PramUkesh
- Paypal: https://paypal.me/PramUkesh
-- Другие способы: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive)
+- Others: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive)
+
#### Спасибо!
-- Здесь вы можете получить больше информации, помощь, прочитать список изменений и исследовать ZeroNet сайты: https://www.reddit.com/r/zeronetx/
-- Общение происходит на канале [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) или в [Gitter](https://gitter.im/canewsin/ZeroNet)
-- Электронная почта: canews.in@gmail.com
+* Больше информации, помощь, журнал изменений, zeronet сайты: https://www.reddit.com/r/zeronetx/
+* Приходите, пообщайтесь с нами: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) или на [gitter](https://gitter.im/canewsin/ZeroNet)
+* Email: canews.in@gmail.com
diff --git a/README.md b/README.md
index 70b79adc..e45d5cad 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
# ZeroNet [](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [](https://hub.docker.com/r/canewsin/zeronet)
+
Decentralized websites using Bitcoin crypto and the BitTorrent network - https://zeronet.dev / [ZeroNet Site](http://127.0.0.1:43110/1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX/), Unlike Bitcoin, ZeroNet Doesn't need a blockchain to run, But uses cryptography used by BTC, to ensure data integrity and validation.
@@ -99,24 +100,6 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/
#### Docker
There is an official image, built from source at: https://hub.docker.com/r/canewsin/zeronet/
-### Online Proxies
-Proxies are like seed boxes for sites(i.e ZNX runs on a cloud vps), you can try zeronet experience from proxies. Add your proxy below if you have one.
-
-#### Official ZNX Proxy :
-
-https://proxy.zeronet.dev/
-
-https://zeronet.dev/
-
-#### From Community
-
-https://0net-preview.com/
-
-https://portal.ngnoid.tv/
-
-https://zeronet.ipfsscan.io/
-
-
### Install from source
- `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip`
diff --git a/build-docker-images.sh b/build-docker-images.sh
new file mode 100755
index 00000000..8eff34f4
--- /dev/null
+++ b/build-docker-images.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+set -e
+
+arg_push=
+
+case "$1" in
+ --push) arg_push=y ; shift ;;
+esac
+
+default_suffix=alpine
+prefix="${1:-local/}"
+
+for dokerfile in dockerfiles/Dockerfile.* ; do
+ suffix="`echo "$dokerfile" | sed 's/.*\/Dockerfile\.//'`"
+ image_name="${prefix}zeronet:$suffix"
+
+ latest=""
+ t_latest=""
+ if [ "$suffix" = "$default_suffix" ] ; then
+ latest="${prefix}zeronet:latest"
+ t_latest="-t ${latest}"
+ fi
+
+ echo "DOCKER BUILD $image_name"
+ docker build -f "$dokerfile" -t "$image_name" $t_latest .
+ if [ -n "$arg_push" ] ; then
+ docker push "$image_name"
+ if [ -n "$latest" ] ; then
+ docker push "$latest"
+ fi
+ fi
+done
diff --git a/dockerfiles/Dockerfile.alpine b/dockerfiles/Dockerfile.alpine
new file mode 120000
index 00000000..0f848cc8
--- /dev/null
+++ b/dockerfiles/Dockerfile.alpine
@@ -0,0 +1 @@
+Dockerfile.alpine3.13
\ No newline at end of file
diff --git a/dockerfiles/Dockerfile.alpine3.13 b/dockerfiles/Dockerfile.alpine3.13
new file mode 100644
index 00000000..79f15b9b
--- /dev/null
+++ b/dockerfiles/Dockerfile.alpine3.13
@@ -0,0 +1,44 @@
+# THIS FILE IS AUTOGENERATED BY gen-dockerfiles.sh.
+# SEE zeronet-Dockerfile FOR THE SOURCE FILE.
+
+FROM alpine:3.13
+
+# Base settings
+ENV HOME /root
+
+# Install packages
+
+# Install packages
+
+COPY install-dep-packages.sh /root/install-dep-packages.sh
+
+RUN /root/install-dep-packages.sh install
+
+COPY requirements.txt /root/requirements.txt
+
+RUN pip3 install -r /root/requirements.txt \
+ && /root/install-dep-packages.sh remove-makedeps \
+ && echo "ControlPort 9051" >> /etc/tor/torrc \
+ && echo "CookieAuthentication 1" >> /etc/tor/torrc
+
+RUN python3 -V \
+ && python3 -m pip list \
+ && tor --version \
+ && openssl version
+
+# Add Zeronet source
+
+COPY . /root
+VOLUME /root/data
+
+# Control if Tor proxy is started
+ENV ENABLE_TOR false
+
+WORKDIR /root
+
+# Set upstart command
+CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
+
+# Expose ports
+EXPOSE 43110 26552
+
diff --git a/dockerfiles/Dockerfile.ubuntu b/dockerfiles/Dockerfile.ubuntu
new file mode 120000
index 00000000..29adf7ef
--- /dev/null
+++ b/dockerfiles/Dockerfile.ubuntu
@@ -0,0 +1 @@
+Dockerfile.ubuntu20.04
\ No newline at end of file
diff --git a/dockerfiles/Dockerfile.ubuntu20.04 b/dockerfiles/Dockerfile.ubuntu20.04
new file mode 100644
index 00000000..bc32cf86
--- /dev/null
+++ b/dockerfiles/Dockerfile.ubuntu20.04
@@ -0,0 +1,44 @@
+# THIS FILE IS AUTOGENERATED BY gen-dockerfiles.sh.
+# SEE zeronet-Dockerfile FOR THE SOURCE FILE.
+
+FROM ubuntu:20.04
+
+# Base settings
+ENV HOME /root
+
+# Install packages
+
+# Install packages
+
+COPY install-dep-packages.sh /root/install-dep-packages.sh
+
+RUN /root/install-dep-packages.sh install
+
+COPY requirements.txt /root/requirements.txt
+
+RUN pip3 install -r /root/requirements.txt \
+ && /root/install-dep-packages.sh remove-makedeps \
+ && echo "ControlPort 9051" >> /etc/tor/torrc \
+ && echo "CookieAuthentication 1" >> /etc/tor/torrc
+
+RUN python3 -V \
+ && python3 -m pip list \
+ && tor --version \
+ && openssl version
+
+# Add Zeronet source
+
+COPY . /root
+VOLUME /root/data
+
+# Control if Tor proxy is started
+ENV ENABLE_TOR false
+
+WORKDIR /root
+
+# Set upstart command
+CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
+
+# Expose ports
+EXPOSE 43110 26552
+
diff --git a/dockerfiles/gen-dockerfiles.sh b/dockerfiles/gen-dockerfiles.sh
new file mode 100755
index 00000000..75a6edf6
--- /dev/null
+++ b/dockerfiles/gen-dockerfiles.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -e
+
+die() {
+ echo "$@" > /dev/stderr
+ exit 1
+}
+
+for os in alpine:3.13 ubuntu:20.04 ; do
+ prefix="`echo "$os" | sed -e 's/://'`"
+ short_prefix="`echo "$os" | sed -e 's/:.*//'`"
+
+ zeronet="zeronet-Dockerfile"
+
+ dockerfile="Dockerfile.$prefix"
+ dockerfile_short="Dockerfile.$short_prefix"
+
+ echo "GEN $dockerfile"
+
+ if ! test -f "$zeronet" ; then
+ die "No such file: $zeronet"
+ fi
+
+ echo "\
+# THIS FILE IS AUTOGENERATED BY gen-dockerfiles.sh.
+# SEE $zeronet FOR THE SOURCE FILE.
+
+FROM $os
+
+`cat "$zeronet"`
+" > "$dockerfile.tmp" && mv "$dockerfile.tmp" "$dockerfile" && ln -s -f "$dockerfile" "$dockerfile_short"
+done
+
diff --git a/Dockerfile.arm64v8 b/dockerfiles/zeronet-Dockerfile
similarity index 55%
rename from Dockerfile.arm64v8
rename to dockerfiles/zeronet-Dockerfile
index d27b7620..92c67c84 100644
--- a/Dockerfile.arm64v8
+++ b/dockerfiles/zeronet-Dockerfile
@@ -1,34 +1,36 @@
-FROM alpine:3.12
-
-#Base settings
+# Base settings
ENV HOME /root
+# Install packages
+
+COPY install-dep-packages.sh /root/install-dep-packages.sh
+
+RUN /root/install-dep-packages.sh install
+
COPY requirements.txt /root/requirements.txt
-#Install ZeroNet
-RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \
- && pip3 install -r /root/requirements.txt \
- && apk del python3-dev gcc libffi-dev musl-dev make \
+RUN pip3 install -r /root/requirements.txt \
+ && /root/install-dep-packages.sh remove-makedeps \
&& echo "ControlPort 9051" >> /etc/tor/torrc \
&& echo "CookieAuthentication 1" >> /etc/tor/torrc
-
+
RUN python3 -V \
&& python3 -m pip list \
&& tor --version \
&& openssl version
-#Add Zeronet source
+# Add Zeronet source
+
COPY . /root
VOLUME /root/data
-#Control if Tor proxy is started
+# Control if Tor proxy is started
ENV ENABLE_TOR false
WORKDIR /root
-#Set upstart command
+# Set upstart command
CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
-#Expose ports
+# Expose ports
EXPOSE 43110 26552
-
diff --git a/install-dep-packages.sh b/install-dep-packages.sh
new file mode 100755
index 00000000..655a33aa
--- /dev/null
+++ b/install-dep-packages.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+set -e
+
+do_alpine() {
+ local deps="python3 py3-pip openssl tor"
+ local makedeps="python3-dev gcc g++ libffi-dev musl-dev make automake autoconf libtool"
+
+ case "$1" in
+ install)
+ apk --update --no-cache --no-progress add $deps $makedeps
+ ;;
+ remove-makedeps)
+ apk del $makedeps
+ ;;
+ esac
+}
+
+do_ubuntu() {
+ local deps="python3 python3-pip openssl tor"
+ local makedeps="python3-dev gcc g++ libffi-dev make automake autoconf libtool"
+
+ case "$1" in
+ install)
+ apt-get update && \
+ apt-get install --no-install-recommends -y $deps $makedeps && \
+ rm -rf /var/lib/apt/lists/*
+ ;;
+ remove-makedeps)
+ apt-get remove -y $makedeps
+ ;;
+ esac
+}
+
+if test -f /etc/os-release ; then
+ . /etc/os-release
+elif test -f /usr/lib/os-release ; then
+ . /usr/lib/os-release
+else
+ echo "No such file: /etc/os-release" > /dev/stderr
+ exit 1
+fi
+
+case "$ID" in
+ ubuntu) do_ubuntu "$@" ;;
+ alpine) do_alpine "$@" ;;
+ *)
+ echo "Unsupported OS ID: $ID" > /dev/stderr
+ exit 1
+esac
diff --git a/plugins b/plugins
deleted file mode 160000
index 689d9309..00000000
--- a/plugins
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 689d9309f73371f4681191b125ec3f2e14075eeb
diff --git a/requirements.txt b/requirements.txt
index 538a6dfc..b3df57ea 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ greenlet==0.4.16; python_version <= "3.6"
gevent>=20.9.0; python_version >= "3.7"
msgpack>=0.4.4
base58
-merkletools @ git+https://github.com/ZeroNetX/pymerkletools.git@dev
+merkletools
rsa
PySocks>=1.6.8
pyasn1
diff --git a/src/Config.py b/src/Config.py
index a9208d55..edc63d26 100644
--- a/src/Config.py
+++ b/src/Config.py
@@ -13,8 +13,8 @@ import time
class Config(object):
def __init__(self, argv):
- self.version = "0.9.0"
- self.rev = 4630
+ self.version = "0.7.6"
+ self.rev = 4565
self.argv = argv
self.action = None
self.test_parser = None
@@ -82,12 +82,45 @@ class Config(object):
from Crypt import CryptHash
access_key_default = CryptHash.random(24, "base64") # Used to allow restrited plugins when multiuser plugin is enabled
trackers = [
+ # by zeroseed at http://127.0.0.1:43110/19HKdTAeBh5nRiKn791czY7TwRB1QNrf1Q/?:users/1HvNGwHKqhj3ZMEM53tz6jbdqe4LRpanEu:zn:dc17f896-bf3f-4962-bdd4-0a470040c9c5
+ "zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441",
+ "zero://2kcb2fqesyaevc4lntogupa4mkdssth2ypfwczd2ov5a3zo6ytwwbayd.onion:15441",
+ "zero://my562dxpjropcd5hy3nd5pemsc4aavbiptci5amwxzbelmzgkkuxpvid.onion:15441",
+ "zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441",
+ "zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441",
+ "zero://tl74auz4tyqv4bieeclmyoe4uwtoc2dj7fdqv4nc4gl5j2bwg2r26bqd.onion:15441",
+ "zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441",
+ "zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441",
+
+ # ZeroNet 0.7.2 defaults:
+ "zero://boot3rdez4rzn36x.onion:15441",
"http://open.acgnxtracker.com:80/announce", # DE
"http://tracker.bt4g.com:2095/announce", # Cloudflare
- "http://tracker.files.fm:6969/announce",
- "http://t.publictracker.xyz:6969/announce",
+ "zero://2602:ffc5::c5b2:5360:26312", # US/ATL
+ "zero://145.239.95.38:15441",
+ "zero://188.116.183.41:26552",
+ "zero://145.239.95.38:15441",
+ "zero://211.125.90.79:22234",
+ "zero://216.189.144.82:26312",
+ "zero://45.77.23.92:15555",
+ "zero://51.15.54.182:21041",
"https://tracker.lilithraws.cf:443/announce",
- "https://tracker.babico.name.tr:443/announce",
+ "udp://code2chicken.nl:6969/announce",
+ "udp://abufinzio.monocul.us:6969/announce",
+ "udp://tracker.0x.tf:6969/announce",
+ "udp://tracker.zerobytes.xyz:1337/announce",
+ "udp://vibe.sleepyinternetfun.xyz:1738/announce",
+ "udp://www.torrent.eu.org:451/announce",
+ "zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441",
+ "zero://2kcb2fqesyaevc4lntogupa4mkdssth2ypfwczd2ov5a3zo6ytwwbayd.onion:15441",
+ "zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441",
+ "zero://hb6ozikfiaafeuqvgseiik4r46szbpjfu66l67wjinnyv6dtopuwhtqd.onion:15445",
+ "zero://75pmmcbp4vvo2zndmjnrkandvbg6jyptygvvpwsf2zguj7urq7t4jzyd.onion:7777",
+ "zero://dw4f4sckg2ultdj5qu7vtkf3jsfxsah3mz6pivwfd6nv3quji3vfvhyd.onion:6969",
+ "zero://5vczpwawviukvd7grfhsfxp7a6huz77hlis4fstjkym5kmf4pu7i7myd.onion:15441",
+ "zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441",
+ "zero://agufghdtniyfwty3wk55drxxwj2zxgzzo7dbrtje73gmvcpxy4ngs4ad.onion:15441",
+ "zero://qn65si4gtcwdiliq7vzrwu62qrweoxb6tx2cchwslaervj6szuje66qd.onion:26117",
]
# Platform specific
if sys.platform.startswith("win"):
@@ -251,12 +284,31 @@ class Config(object):
self.parser.add_argument('--access_key', help='Plugin access key default: Random key generated at startup', default=access_key_default, metavar='key')
self.parser.add_argument('--dist_type', help='Type of installed distribution', default='source')
- self.parser.add_argument('--size_limit', help='Default site size limit in MB', default=25, type=int, metavar='limit')
+ self.parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, type=int, metavar='limit')
self.parser.add_argument('--file_size_limit', help='Maximum per file size limit in MB', default=10, type=int, metavar='limit')
- self.parser.add_argument('--connected_limit', help='Max connected peer per site', default=8, type=int, metavar='connected_limit')
- self.parser.add_argument('--global_connected_limit', help='Max connections', default=512, type=int, metavar='global_connected_limit')
+ self.parser.add_argument('--connected_limit', help='Max number of connected peers per site. Soft limit.', default=10, type=int, metavar='connected_limit')
+ self.parser.add_argument('--global_connected_limit', help='Max number of connections. Soft limit.', default=512, type=int, metavar='global_connected_limit')
self.parser.add_argument('--workers', help='Download workers per site', default=5, type=int, metavar='workers')
+ self.parser.add_argument('--site_announce_interval_min', help='Site announce interval for the most active sites, in minutes.', default=4, type=int, metavar='site_announce_interval_min')
+ self.parser.add_argument('--site_announce_interval_max', help='Site announce interval for inactive sites, in minutes.', default=30, type=int, metavar='site_announce_interval_max')
+
+ self.parser.add_argument('--site_peer_check_interval_min', help='Connectable peers check interval for the most active sites, in minutes.', default=5, type=int, metavar='site_peer_check_interval_min')
+ self.parser.add_argument('--site_peer_check_interval_max', help='Connectable peers check interval for inactive sites, in minutes.', default=20, type=int, metavar='site_peer_check_interval_max')
+
+ self.parser.add_argument('--site_update_check_interval_min', help='Site update check interval for the most active sites, in minutes.', default=5, type=int, metavar='site_update_check_interval_min')
+ self.parser.add_argument('--site_update_check_interval_max', help='Site update check interval for inactive sites, in minutes.', default=45, type=int, metavar='site_update_check_interval_max')
+
+ self.parser.add_argument('--site_connectable_peer_count_max', help='Search for as many connectable peers for the most active sites', default=10, type=int, metavar='site_connectable_peer_count_max')
+ self.parser.add_argument('--site_connectable_peer_count_min', help='Search for as many connectable peers for inactive sites', default=2, type=int, metavar='site_connectable_peer_count_min')
+
+ self.parser.add_argument('--send_back_lru_size', help='Size of the send back LRU cache', default=5000, type=int, metavar='send_back_lru_size')
+ self.parser.add_argument('--send_back_limit', help='Send no more than so many files at once back to peer, when we discovered that the peer held older file versions', default=3, type=int, metavar='send_back_limit')
+
+ self.parser.add_argument('--expose_no_ownership', help='By default, ZeroNet tries checking updates for own sites more frequently. This can be used by a third party for revealing the network addresses of a site owner. If this option is enabled, ZeroNet performs the checks in the same way for any sites.', type='bool', choices=[True, False], default=False)
+
+ self.parser.add_argument('--simultaneous_connection_throttle_threshold', help='Throttle opening new connections when the number of outgoing connections in not fully established state exceeds the threshold.', default=15, type=int, metavar='simultaneous_connection_throttle_threshold')
+
self.parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
self.parser.add_argument('--fileserver_port', help='FileServer bind port (0: randomize)', default=0, type=int, metavar='port')
self.parser.add_argument('--fileserver_port_range', help='FileServer randomization range', default="10000-40000", metavar='port')
@@ -319,7 +371,8 @@ class Config(object):
def loadTrackersFile(self):
if not self.trackers_file:
- self.trackers_file = ["trackers.txt", "{data_dir}/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d/trackers.txt"]
+ return None
+
self.trackers = self.arguments.trackers[:]
for trackers_file in self.trackers_file:
@@ -331,9 +384,6 @@ class Config(object):
else: # Relative to zeronet.py
trackers_file_path = self.start_dir + "/" + trackers_file
- if not os.path.exists(trackers_file_path):
- continue
-
for line in open(trackers_file_path):
tracker = line.strip()
if "://" in tracker and tracker not in self.trackers:
diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py
index 22bcf29c..8916afbe 100644
--- a/src/Connection/Connection.py
+++ b/src/Connection/Connection.py
@@ -17,12 +17,13 @@ from util import helper
class Connection(object):
__slots__ = (
"sock", "sock_wrapped", "ip", "port", "cert_pin", "target_onion", "id", "protocol", "type", "server", "unpacker", "unpacker_bytes", "req_id", "ip_type",
- "handshake", "crypt", "connected", "event_connected", "closed", "start_time", "handshake_time", "last_recv_time", "is_private_ip", "is_tracker_connection",
+ "handshake", "crypt", "connected", "connecting", "event_connected", "closed", "start_time", "handshake_time", "last_recv_time", "is_private_ip", "is_tracker_connection",
"last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent", "cpu_time", "send_lock",
"last_ping_delay", "last_req_time", "last_cmd_sent", "last_cmd_recv", "bad_actions", "sites", "name", "waiting_requests", "waiting_streams"
)
def __init__(self, server, ip, port, sock=None, target_onion=None, is_tracker_connection=False):
+ self.server = server
self.sock = sock
self.cert_pin = None
if "#" in ip:
@@ -42,7 +43,6 @@ class Connection(object):
self.is_private_ip = False
self.is_tracker_connection = is_tracker_connection
- self.server = server
self.unpacker = None # Stream incoming socket messages here
self.unpacker_bytes = 0 # How many bytes the unpacker received
self.req_id = 0 # Last request id
@@ -50,6 +50,7 @@ class Connection(object):
self.crypt = None # Connection encryption method
self.sock_wrapped = False # Socket wrapped to encryption
+ self.connecting = False
self.connected = False
self.event_connected = gevent.event.AsyncResult() # Solves on handshake received
self.closed = False
@@ -81,11 +82,11 @@ class Connection(object):
def setIp(self, ip):
self.ip = ip
- self.ip_type = helper.getIpType(ip)
+ self.ip_type = self.server.getIpType(ip)
self.updateName()
def createSocket(self):
- if helper.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"):
+ if self.server.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"):
# Create IPv6 connection as IPv4 when using proxy
return socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
@@ -118,13 +119,28 @@ class Connection(object):
# Open connection to peer and wait for handshake
def connect(self):
+ self.connecting = True
+ try:
+ return self._connect()
+ except Exception as err:
+ self.connecting = False
+ self.connected = False
+ raise
+
+ def _connect(self):
+ self.updateOnlineStatus(outgoing_activity=True)
+
+ if not self.event_connected or self.event_connected.ready():
+ self.event_connected = gevent.event.AsyncResult()
+
self.type = "out"
+
+ unreachability = self.server.getIpUnreachability(self.ip)
+ if unreachability:
+ raise Exception(unreachability)
+
if self.ip_type == "onion":
- if not self.server.tor_manager or not self.server.tor_manager.enabled:
- raise Exception("Can't connect to onion addresses, no Tor controller present")
self.sock = self.server.tor_manager.createSocket(self.ip, self.port)
- elif config.tor == "always" and helper.isPrivateIp(self.ip) and self.ip not in config.ip_local:
- raise Exception("Can't connect to local IPs in Tor: always mode")
elif config.trackers_proxy != "disable" and config.tor != "always" and self.is_tracker_connection:
if config.trackers_proxy == "tor":
self.sock = self.server.tor_manager.createSocket(self.ip, self.port)
@@ -148,37 +164,56 @@ class Connection(object):
self.sock.connect(sock_address)
- # Implicit SSL
- should_encrypt = not self.ip_type == "onion" and self.ip not in self.server.broken_ssl_ips and self.ip not in config.ip_local
- if self.cert_pin:
- self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", cert_pin=self.cert_pin)
- self.sock.do_handshake()
- self.crypt = "tls-rsa"
- self.sock_wrapped = True
- elif should_encrypt and "tls-rsa" in CryptConnection.manager.crypt_supported:
+ if self.shouldEncrypt():
try:
- self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa")
- self.sock.do_handshake()
- self.crypt = "tls-rsa"
- self.sock_wrapped = True
+ self.wrapSocket()
except Exception as err:
- if not config.force_encryption:
- self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err)))
- self.server.broken_ssl_ips[self.ip] = True
- self.sock.close()
- self.crypt = None
- self.sock = self.createSocket()
- self.sock.settimeout(30)
- self.sock.connect(sock_address)
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ if self.mustEncrypt():
+ raise
+ self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err)))
+ self.server.broken_ssl_ips[self.ip] = True
+ return self.connect()
# Detect protocol
- self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()})
event_connected = self.event_connected
- gevent.spawn(self.messageLoop)
+ self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()})
+ self.server.outgoing_pool.spawn(self.messageLoop)
connect_res = event_connected.get() # Wait for handshake
- self.sock.settimeout(timeout_before)
+ if self.sock:
+ self.sock.settimeout(timeout_before)
return connect_res
+ def mustEncrypt(self):
+ if self.cert_pin:
+ return True
+ if (not self.ip_type == "onion") and config.force_encryption:
+ return True
+ return False
+
+ def shouldEncrypt(self):
+ if self.mustEncrypt():
+ return True
+ return (
+ (not self.ip_type == "onion")
+ and
+ (self.ip not in self.server.broken_ssl_ips)
+ and
+ (self.ip not in config.ip_local)
+ and
+ ("tls-rsa" in CryptConnection.manager.crypt_supported)
+ )
+
+ def wrapSocket(self, crypt="tls-rsa", do_handshake=True):
+ server = (self.type == "in")
+ sock = CryptConnection.manager.wrapSocket(self.sock, crypt, server=server, cert_pin=self.cert_pin)
+ sock.do_handshake()
+ self.crypt = crypt
+ self.sock_wrapped = True
+ self.sock = sock
+
# Handle incoming connection
def handleIncomingConnection(self, sock):
self.log("Incoming connection...")
@@ -192,9 +227,7 @@ class Connection(object):
first_byte = sock.recv(1, gevent.socket.MSG_PEEK)
if first_byte == b"\x16":
self.log("Crypt in connection using implicit SSL")
- self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", True)
- self.sock_wrapped = True
- self.crypt = "tls-rsa"
+ self.wrapSocket(do_handshake=False)
except Exception as err:
self.log("Socket peek error: %s" % Debug.formatException(err))
self.messageLoop()
@@ -213,6 +246,7 @@ class Connection(object):
self.protocol = "v2"
self.updateName()
self.connected = True
+ self.connecting = False
buff_len = 0
req_len = 0
self.unpacker_bytes = 0
@@ -435,13 +469,13 @@ class Connection(object):
self.updateName()
self.event_connected.set(True) # Mark handshake as done
- self.event_connected = None
self.handshake_time = time.time()
# Handle incoming message
def handleMessage(self, message):
cmd = message["cmd"]
+ self.updateOnlineStatus(successful_activity=True)
self.last_message_time = time.time()
self.last_cmd_recv = cmd
if cmd == "response": # New style response
@@ -458,12 +492,10 @@ class Connection(object):
self.last_ping_delay = ping
# Server switched to crypt, lets do it also if not crypted already
if message.get("crypt") and not self.sock_wrapped:
- self.crypt = message["crypt"]
+ crypt = message["crypt"]
server = (self.type == "in")
- self.log("Crypt out connection using: %s (server side: %s, ping: %.3fs)..." % (self.crypt, server, ping))
- self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin)
- self.sock.do_handshake()
- self.sock_wrapped = True
+ self.log("Crypt out connection using: %s (server side: %s, ping: %.3fs)..." % (crypt, server, ping))
+ self.wrapSocket(crypt)
if not self.sock_wrapped and self.cert_pin:
self.close("Crypt connection error: Socket not encrypted, but certificate pin present")
@@ -491,8 +523,7 @@ class Connection(object):
server = (self.type == "in")
self.log("Crypt in connection using: %s (server side: %s)..." % (self.crypt, server))
try:
- self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin)
- self.sock_wrapped = True
+ self.wrapSocket(self.crypt)
except Exception as err:
if not config.force_encryption:
self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err)))
@@ -504,6 +535,7 @@ class Connection(object):
# Send data to connection
def send(self, message, streaming=False):
+ self.updateOnlineStatus(outgoing_activity=True)
self.last_send_time = time.time()
if config.debug_socket:
self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (
@@ -543,6 +575,11 @@ class Connection(object):
message = None
with self.send_lock:
self.sock.sendall(data)
+ # XXX: Should not be used here:
+ # self.updateOnlineStatus(successful_activity=True)
+ # Looks like self.sock.sendall() returns normally, instead of
+ # raising an Exception (at least, some times).
+ # So the only way of detecting the network activity is self.handleMessage()
except Exception as err:
self.close("Send error: %s (cmd: %s)" % (err, stat_key))
return False
@@ -554,7 +591,7 @@ class Connection(object):
buff = 64 * 1024
bytes_left = read_bytes
bytes_sent = 0
- while True:
+ while True and self.sock != None:
self.last_send_time = time.time()
data = file.read(min(bytes_left, buff))
bytes_sent += len(data)
@@ -584,7 +621,8 @@ class Connection(object):
self.waiting_requests[self.req_id] = {"evt": event, "cmd": cmd}
if stream_to:
self.waiting_streams[self.req_id] = stream_to
- self.send(data) # Send request
+ if not self.send(data): # Send request
+ return False
res = event.get() # Wait until event solves
return res
@@ -608,6 +646,7 @@ class Connection(object):
return False # Already closed
self.closed = True
self.connected = False
+ self.connecting = False
if self.event_connected:
self.event_connected.set(False)
@@ -633,3 +672,12 @@ class Connection(object):
self.sock = None
self.unpacker = None
self.event_connected = None
+ self.crypt = None
+ self.sock_wrapped = False
+
+ return True
+
+ def updateOnlineStatus(self, outgoing_activity=False, successful_activity=False):
+ self.server.updateOnlineStatus(self,
+ outgoing_activity=outgoing_activity,
+ successful_activity=successful_activity)
diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py
index c9048398..5f5e7a29 100644
--- a/src/Connection/ConnectionServer.py
+++ b/src/Connection/ConnectionServer.py
@@ -1,4 +1,5 @@
import logging
+import re
import time
import sys
import socket
@@ -8,6 +9,7 @@ import gevent
import msgpack
from gevent.server import StreamServer
from gevent.pool import Pool
+import gevent.event
import util
from util import helper
@@ -32,25 +34,36 @@ class ConnectionServer(object):
self.port = port
self.last_connection_id = 0 # Connection id incrementer
self.last_connection_id_current_version = 0 # Connection id incrementer for current client version
- self.last_connection_id_supported_version = 0 # Connection id incrementer for last supported version
self.log = logging.getLogger("ConnServer")
self.port_opened = {}
self.peer_blacklist = SiteManager.peer_blacklist
+ self.managed_pools = {}
+
self.tor_manager = TorManager(self.ip, self.port)
self.connections = [] # Connections
self.whitelist = config.ip_local # No flood protection on this ips
self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood
self.broken_ssl_ips = {} # Peerids of broken ssl connections
self.ips = {} # Connection by ip
+
self.has_internet = True # Internet outage detection
+ self.internet_online_since = 0
+ self.internet_offline_since = 0
+ self.last_outgoing_internet_activity_time = 0 # Last time the application tried to send any data
+ self.last_successful_internet_activity_time = 0 # Last time the application successfully sent or received any data
+ self.internet_outage_threshold = 60 * 2
self.stream_server = None
self.stream_server_proxy = None
self.running = False
self.stopping = False
+ self.stopping_event = gevent.event.Event()
self.thread_checker = None
+ self.thread_pool = Pool(None)
+ self.managed_pools["thread"] = self.thread_pool
+
self.stat_recv = defaultdict(lambda: defaultdict(int))
self.stat_sent = defaultdict(lambda: defaultdict(int))
self.bytes_recv = 0
@@ -62,8 +75,14 @@ class ConnectionServer(object):
self.num_outgoing = 0
self.had_external_incoming = False
+
+
self.timecorrection = 0.0
self.pool = Pool(500) # do not accept more than 500 connections
+ self.managed_pools["incoming"] = self.pool
+
+ self.outgoing_pool = Pool(None)
+ self.managed_pools["outgoing"] = self.outgoing_pool
# Bittorrent style peerid
self.peer_id = "-UT3530-%s" % CryptHash.random(12, "base64")
@@ -84,10 +103,11 @@ class ConnectionServer(object):
return False
self.running = True
if check_connections:
- self.thread_checker = gevent.spawn(self.checkConnections)
+ self.thread_checker = self.spawn(self.checkConnections)
CryptConnection.manager.loadCerts()
if config.tor != "disable":
self.tor_manager.start()
+ self.tor_manager.startOnions()
if not self.port:
self.log.info("No port found, not binding")
return False
@@ -108,7 +128,7 @@ class ConnectionServer(object):
return None
if self.stream_server_proxy:
- gevent.spawn(self.listenProxy)
+ self.spawn(self.listenProxy)
try:
self.stream_server.serve_forever()
except Exception as err:
@@ -116,22 +136,92 @@ class ConnectionServer(object):
return False
self.log.debug("Stopped.")
- def stop(self):
+ def stop(self, ui_websocket=None):
self.log.debug("Stopping %s" % self.stream_server)
self.stopping = True
self.running = False
+ self.stopping_event.set()
+ self.onStop(ui_websocket=ui_websocket)
+
+ def onStop(self, ui_websocket=None):
+ timeout = 30
+ start_time = time.time()
+ join_quantum = 0.1
+ prev_msg = None
+ while True:
+ if time.time() >= start_time + timeout:
+ break
+
+ total_size = 0
+ sizes = {}
+ timestep = 0
+ for name, pool in list(self.managed_pools.items()):
+ timestep += join_quantum
+ pool.join(timeout=join_quantum)
+ size = len(pool)
+ if size:
+ sizes[name] = size
+ total_size += size
+
+ if len(sizes) == 0:
+ break
+
+ if timestep < 1:
+ time.sleep(1 - timestep)
+
+ # format message
+ s = ""
+ for name, size in sizes.items():
+ s += "%s pool: %s, " % (name, size)
+ msg = "Waiting for tasks in managed pools to stop: %s" % s
+ # Prevent flooding to log
+ if msg != prev_msg:
+ prev_msg = msg
+ self.log.info("%s", msg)
+
+ percent = 100 * (time.time() - start_time) / timeout
+ msg = "File Server: waiting for %s tasks to stop" % total_size
+ self.sendShutdownProgress(ui_websocket, msg, percent)
+
+ for name, pool in list(self.managed_pools.items()):
+ size = len(pool)
+ if size:
+ self.log.info("Killing %s tasks in %s pool", size, name)
+ pool.kill()
+
+ self.sendShutdownProgress(ui_websocket, "File Server stopped. Now to exit.", 100)
+
if self.thread_checker:
gevent.kill(self.thread_checker)
+ self.thread_checker = None
if self.stream_server:
self.stream_server.stop()
+ def sendShutdownProgress(self, ui_websocket, message, progress):
+ if not ui_websocket:
+ return
+ ui_websocket.cmd("progress", ["shutdown", message, progress])
+ time.sleep(0.01)
+
+ # Sleeps the specified amount of time or until ConnectionServer is stopped
+ def sleep(self, t):
+ if t:
+ self.stopping_event.wait(timeout=t)
+ else:
+ time.sleep(t)
+
+ # Spawns a thread that will be waited for on server being stopped (and killed after a timeout)
+ def spawn(self, *args, **kwargs):
+ thread = self.thread_pool.spawn(*args, **kwargs)
+ return thread
+
def closeConnections(self):
self.log.debug("Closing all connection: %s" % len(self.connections))
for connection in self.connections[:]:
connection.close("Close all connections")
def handleIncomingConnection(self, sock, addr):
- if config.offline:
+ if not self.allowsAcceptingConnections():
sock.close()
return False
@@ -149,7 +239,7 @@ class ConnectionServer(object):
self.ip_incoming[ip] += 1
if self.ip_incoming[ip] > 6: # Allow 6 in 1 minute from same ip
self.log.debug("Connection flood detected from %s" % ip)
- time.sleep(30)
+ self.sleep(30)
sock.close()
return False
else:
@@ -158,10 +248,8 @@ class ConnectionServer(object):
connection = Connection(self, ip, port, sock)
self.connections.append(connection)
rev = connection.handshake.get("rev", 0)
- if rev >= 4560:
- self.last_connection_id_supported_version += 1
- if rev == config.rev:
- self.last_connection_id_current_version += 1
+ if rev > 0 and rev == config.rev:
+ self.last_connection_id_current_version += 1
if ip not in config.ip_local:
self.ips[ip] = connection
connection.handleIncomingConnection(sock)
@@ -170,7 +258,7 @@ class ConnectionServer(object):
pass
def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None, is_tracker_connection=False):
- ip_type = helper.getIpType(ip)
+ ip_type = self.getIpType(ip)
has_per_site_onion = (ip.endswith(".onion") or self.port_opened.get(ip_type, None) == False) and self.tor_manager.start_onions and site
if has_per_site_onion: # Site-unique connection for Tor
if ip.endswith(".onion"):
@@ -206,7 +294,7 @@ class ConnectionServer(object):
return connection
# No connection found
- if create and not config.offline: # Allow to create new connection if not found
+ if create and self.allowsCreatingConnections():
if port == 0:
raise Exception("This peer is not connectable")
@@ -214,6 +302,7 @@ class ConnectionServer(object):
raise Exception("This peer is blacklisted")
try:
+ #self.log.info("Connection to: %s:%s", ip, port)
if has_per_site_onion: # Lock connection to site
connection = Connection(self, ip, port, target_onion=site_onion, is_tracker_connection=is_tracker_connection)
else:
@@ -228,17 +317,16 @@ class ConnectionServer(object):
raise Exception("Connection event return error")
else:
rev = connection.handshake.get("rev", 0)
- if rev >= 4560:
- self.last_connection_id_supported_version += 1
- if rev == config.rev:
- self.last_connection_id_current_version += 1
+ if rev > 0 and rev == config.rev:
+ self.last_connection_id_current_version += 1
except Exception as err:
+ #self.log.info("Connection error (%s, %s): %s", ip, port, Debug.formatException(err))
connection.close("%s Connect error: %s" % (ip, Debug.formatException(err)))
raise err
if len(self.connections) > config.global_connected_limit:
- gevent.spawn(self.checkMaxConnections)
+ self.spawn(self.checkMaxConnections)
return connection
else:
@@ -261,12 +349,12 @@ class ConnectionServer(object):
def checkConnections(self):
run_i = 0
- time.sleep(15)
+ self.sleep(15)
while self.running:
run_i += 1
self.ip_incoming = {} # Reset connected ips counter
- last_message_time = 0
s = time.time()
+ self.updateOnlineStatus(None)
for connection in self.connections[:]: # Make a copy
if connection.ip.endswith(".onion") or config.tor == "always":
timeout_multipler = 2
@@ -274,9 +362,6 @@ class ConnectionServer(object):
timeout_multipler = 1
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
- if connection.last_message_time > last_message_time and not connection.is_private_ip:
- # Message from local IPs does not means internet connection
- last_message_time = connection.last_message_time
if connection.unpacker and idle > 30:
# Delete the unpacker if not needed
@@ -324,24 +409,12 @@ class ConnectionServer(object):
# Reset bad action counter every 30 min
connection.bad_actions = 0
- # Internet outage detection
- if time.time() - last_message_time > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)):
- # Offline: Last message more than 60-600sec depending on connection number
- if self.has_internet and last_message_time:
- self.has_internet = False
- self.onInternetOffline()
- else:
- # Online
- if not self.has_internet:
- self.has_internet = True
- self.onInternetOnline()
-
self.timecorrection = self.getTimecorrection()
if time.time() - s > 0.01:
self.log.debug("Connection cleanup in %.3fs" % (time.time() - s))
- time.sleep(15)
+ self.sleep(15)
self.log.debug("Checkconnections ended")
@util.Noparallel(blocking=False)
@@ -366,6 +439,68 @@ class ConnectionServer(object):
))
return num_closed
+ # Returns True if we should slow down opening new connections as at the moment
+ # there are too many connections being established and not connected completely
+ # (not entered the message loop yet).
+ def shouldThrottleNewConnections(self):
+ threshold = config.simultaneous_connection_throttle_threshold
+ if len(self.connections) <= threshold:
+ return False
+ nr_connections_being_established = 0
+ for connection in self.connections[:]: # Make a copy
+ if connection.connecting and not connection.connected and connection.type == "out":
+ nr_connections_being_established += 1
+ if nr_connections_being_established > threshold:
+ return True
+ return False
+
+ # Internet outage detection
+ def updateOnlineStatus(self, connection, outgoing_activity=False, successful_activity=False):
+
+ now = time.time()
+
+ if connection and not connection.is_private_ip:
+ if outgoing_activity:
+ self.last_outgoing_internet_activity_time = now
+ if successful_activity:
+ self.last_successful_internet_activity_time = now
+ self.setInternetStatus(True)
+ return
+
+ if not self.last_outgoing_internet_activity_time:
+ return
+
+ if (
+ (self.last_successful_internet_activity_time < now - self.internet_outage_threshold)
+ and
+ (self.last_successful_internet_activity_time < self.last_outgoing_internet_activity_time)
+ ):
+ self.setInternetStatus(False)
+ return
+
+ # This is the old algorithm just in case we missed something
+ idle = now - self.last_successful_internet_activity_time
+ if idle > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)):
+ # Offline: Last successful activity more than 60-600sec depending on connection number
+ self.setInternetStatus(False)
+ return
+
+ def setInternetStatus(self, status):
+ if self.has_internet == status:
+ return
+
+ self.has_internet = status
+
+ if self.has_internet:
+ self.internet_online_since = time.time()
+ self.spawn(self.onInternetOnline)
+ else:
+ self.internet_offline_since = time.time()
+ self.spawn(self.onInternetOffline)
+
+ def isInternetOnline(self):
+ return self.has_internet
+
def onInternetOnline(self):
self.log.info("Internet online")
@@ -373,6 +508,32 @@ class ConnectionServer(object):
self.had_external_incoming = False
self.log.info("Internet offline")
+ def setOfflineMode(self, offline_mode):
+ if config.offline == offline_mode:
+ return
+ config.offline = offline_mode # Yep, awkward
+ if offline_mode:
+ self.log.info("offline mode is ON")
+ else:
+ self.log.info("offline mode is OFF")
+
+ def isOfflineMode(self):
+ return config.offline
+
+ def allowsCreatingConnections(self):
+ if self.isOfflineMode():
+ return False
+ if self.stopping:
+ return False
+ return True
+
+ def allowsAcceptingConnections(self):
+ if self.isOfflineMode():
+ return False
+ if self.stopping:
+ return False
+ return True
+
def getTimecorrection(self):
corrections = sorted([
connection.handshake.get("time") - connection.handshake_time + connection.last_ping_delay
@@ -384,3 +545,48 @@ class ConnectionServer(object):
mid = int(len(corrections) / 2 - 1)
median = (corrections[mid - 1] + corrections[mid] + corrections[mid + 1]) / 3
return median
+
+
+ ############################################################################
+
+ # Methods for handling network address types
+ # (ipv4, ipv6, onion etc... more to be implemented by plugins)
+ #
+ # All the functions handling network address types have "Ip" in the name.
+ # So it was in the initial codebase, and I keep the naming, since I couldn't
+ # think of a better option.
+ # "IP" is short and quite clear and lets you understand that a variable
+ # contains a peer address or other transport-level address and not
+ # an address of ZeroNet site.
+ #
+
+ # Returns type of the given network address.
+ # Since: 0.8.0
+ # Replaces helper.getIpType() in order to be extensible by plugins.
+ def getIpType(self, ip):
+ if ip.endswith(".onion"):
+ return "onion"
+ elif ":" in ip:
+ return "ipv6"
+ elif re.match(r"[0-9\.]+$", ip):
+ return "ipv4"
+ else:
+ return "unknown"
+
+ # Checks if a network address can be reachable in the current configuration
+ # and returs a string describing why it cannot.
+ # If the network address can be reachable, returns False.
+ # Since: 0.8.0
+ def getIpUnreachability(self, ip):
+ ip_type = self.getIpType(ip)
+ if ip_type == 'onion' and not self.tor_manager.enabled:
+ return "Can't connect to onion addresses, no Tor controller present"
+ if config.tor == "always" and helper.isPrivateIp(ip) and ip not in config.ip_local:
+ return "Can't connect to local IPs in Tor: always mode"
+ return False
+
+ # Returns True if ConnctionServer has means for establishing outgoing
+ # connections to the given address.
+ # Since: 0.8.0
+ def isIpReachable(self, ip):
+ return self.getIpUnreachability(ip) == False
diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py
index 623cc707..c6a64750 100644
--- a/src/Content/ContentManager.py
+++ b/src/Content/ContentManager.py
@@ -239,7 +239,7 @@ class ContentManager(object):
if num_removed_bad_files > 0:
self.site.worker_manager.removeSolvedFileTasks(mark_as_good=False)
- gevent.spawn(self.site.update, since=0)
+ self.site.spawn(self.site.update, since=0)
self.log.debug("Archived removed contents: %s, removed bad files: %s" % (num_removed_contents, num_removed_bad_files))
@@ -651,6 +651,25 @@ class ContentManager(object):
)
return files_node, files_optional_node
+ def serializeForSigning(self, content):
+ if "sign" in content:
+ del(content["sign"]) # The file signed without the sign
+ if "signs" in content:
+ del(content["signs"]) # The file signed without the signs
+
+ sign_content = json.dumps(content, sort_keys=True) # Dump the json to string to remove whitespaces
+
+ # Fix float representation error on Android
+ modified = content["modified"]
+ if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"):
+ modified_fixed = "{:.6f}".format(modified).strip("0.")
+ sign_content = sign_content.replace(
+ '"modified": %s' % repr(modified),
+ '"modified": %s' % modified_fixed
+ )
+
+ return sign_content
+
# Create and sign a content.json
# Return: The new content if filewrite = False
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None, remove_missing_optional=False):
@@ -727,6 +746,7 @@ class ContentManager(object):
elif "files_optional" in new_content:
del new_content["files_optional"]
+ new_content["modified"] = int(time.time()) # Add timestamp
if inner_path == "content.json":
new_content["zeronet_version"] = config.version
new_content["signs_required"] = content.get("signs_required", 1)
@@ -746,44 +766,20 @@ class ContentManager(object):
)
self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers))
- signs_required = 1
if inner_path == "content.json" and privatekey_address == self.site.address:
# If signing using the root key, then sign the valid signers
- signs_required = new_content["signs_required"]
- signers_data = "%s:%s" % (signs_required, ",".join(valid_signers))
+ signers_data = "%s:%s" % (new_content["signs_required"], ",".join(valid_signers))
new_content["signers_sign"] = CryptBitcoin.sign(str(signers_data), privatekey)
if not new_content["signers_sign"]:
self.log.info("Old style address, signers_sign is none")
self.log.info("Signing %s..." % inner_path)
- if "signs" in new_content:
- # del(new_content["signs"]) # Delete old signs
- old_signs_content = new_content["signs"]
- del(new_content["signs"])
- else:
- old_signs_content = None
- if "sign" in new_content:
- del(new_content["sign"]) # Delete old sign (backward compatibility)
-
- if signs_required > 1:
- has_valid_sign = False
- sign_content = json.dumps(new_content, sort_keys=True)
- for signer in valid_signers:
- res = CryptBitcoin.verify(sign_content,signer,old_signs_content[signer]);
- print(res)
- if res:
- has_valid_sign = has_valid_sign or res
- if has_valid_sign:
- new_content["modified"] = content["modified"]
- sign_content = json.dumps(new_content, sort_keys=True)
- else:
- new_content["modified"] = int(time.time()) # Add timestamp
- sign_content = json.dumps(new_content, sort_keys=True)
+ sign_content = self.serializeForSigning(new_content)
sign = CryptBitcoin.sign(sign_content, privatekey)
# new_content["signs"] = content.get("signs", {}) # TODO: Multisig
if sign: # If signing is successful (not an old address)
- new_content["signs"] = old_signs_content or {}
+ new_content["signs"] = {}
new_content["signs"][privatekey_address] = sign
self.verifyContent(inner_path, new_content)
@@ -818,9 +814,7 @@ class ContentManager(object):
# Return: The required number of valid signs for the content.json
def getSignsRequired(self, inner_path, content=None):
- if not content:
- return 1
- return content.get("signs_required", 1)
+ return 1 # Todo: Multisig
def verifyCertSign(self, user_address, user_auth_type, user_name, issuer_address, sign):
from Crypt import CryptBitcoin
@@ -945,104 +939,95 @@ class ContentManager(object):
return True # All good
+ def verifyContentFile(self, inner_path, file, ignore_same=True):
+ from Crypt import CryptBitcoin
+
+ if type(file) is dict:
+ new_content = file
+ else:
+ try:
+ if sys.version_info.major == 3 and sys.version_info.minor < 6:
+ new_content = json.loads(file.read().decode("utf8"))
+ else:
+ new_content = json.load(file)
+ except Exception as err:
+ raise VerifyError("Invalid json file: %s" % err)
+ if inner_path in self.contents:
+ old_content = self.contents.get(inner_path, {"modified": 0})
+ # Checks if its newer the ours
+ if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
+ return None
+ elif old_content["modified"] > new_content["modified"]: # We have newer
+ raise VerifyError(
+ "We have newer (Our: %s, Sent: %s)" %
+ (old_content["modified"], new_content["modified"])
+ )
+ if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+)
+ raise VerifyError("Modify timestamp is in the far future!")
+ if self.isArchived(inner_path, new_content["modified"]):
+ if inner_path in self.site.bad_files:
+ del self.site.bad_files[inner_path]
+ raise VerifyError("This file is archived!")
+ # Check sign
+ sign = new_content.get("sign")
+ signs = new_content.get("signs", {})
+ sign_content = self.serializeForSigning(new_content)
+
+ if signs: # New style signing
+ valid_signers = self.getValidSigners(inner_path, new_content)
+ signs_required = self.getSignsRequired(inner_path, new_content)
+
+ if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
+ signers_data = "%s:%s" % (signs_required, ",".join(valid_signers))
+ if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]):
+ raise VerifyError("Invalid signers_sign!")
+
+ if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
+ raise VerifyError("Invalid cert!")
+
+ valid_signs = 0
+ for address in valid_signers:
+ if address in signs:
+ valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
+ if valid_signs >= signs_required:
+ break # Break if we has enough signs
+ if valid_signs < signs_required:
+ raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required))
+ else:
+ return self.verifyContent(inner_path, new_content)
+ elif sign: # Old style signing
+ raise VerifyError("Invalid old-style sign")
+ else:
+ raise VerifyError("Not signed")
+
+ def verifyOrdinaryFile(self, inner_path, file, ignore_same=True):
+ file_info = self.getFileInfo(inner_path)
+ if file_info:
+ if CryptHash.sha512sum(file) != file_info.get("sha512", ""):
+ raise VerifyError("Invalid hash")
+
+ if file_info.get("size", 0) != file.tell():
+ raise VerifyError(
+ "File size does not match %s <> %s" %
+ (inner_path, file.tell(), file_info.get("size", 0))
+ )
+
+ return True
+
+ else: # File not in content.json
+ raise VerifyError("File not in content.json")
+
# Verify file validity
# Return: None = Same as before, False = Invalid, True = Valid
def verifyFile(self, inner_path, file, ignore_same=True):
- if inner_path.endswith("content.json"): # content.json: Check using sign
- from Crypt import CryptBitcoin
- try:
- if type(file) is dict:
- new_content = file
- else:
- try:
- if sys.version_info.major == 3 and sys.version_info.minor < 6:
- new_content = json.loads(file.read().decode("utf8"))
- else:
- new_content = json.load(file)
- except Exception as err:
- raise VerifyError("Invalid json file: %s" % err)
- if inner_path in self.contents:
- old_content = self.contents.get(inner_path, {"modified": 0})
- # Checks if its newer the ours
- if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
- return None
- elif old_content["modified"] > new_content["modified"]: # We have newer
- raise VerifyError(
- "We have newer (Our: %s, Sent: %s)" %
- (old_content["modified"], new_content["modified"])
- )
- if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+)
- raise VerifyError("Modify timestamp is in the far future!")
- if self.isArchived(inner_path, new_content["modified"]):
- if inner_path in self.site.bad_files:
- del self.site.bad_files[inner_path]
- raise VerifyError("This file is archived!")
- # Check sign
- sign = new_content.get("sign")
- signs = new_content.get("signs", {})
- if "sign" in new_content:
- del(new_content["sign"]) # The file signed without the sign
- if "signs" in new_content:
- del(new_content["signs"]) # The file signed without the signs
-
- sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
-
- # Fix float representation error on Android
- modified = new_content["modified"]
- if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"):
- modified_fixed = "{:.6f}".format(modified).strip("0.")
- sign_content = sign_content.replace(
- '"modified": %s' % repr(modified),
- '"modified": %s' % modified_fixed
- )
-
- if signs: # New style signing
- valid_signers = self.getValidSigners(inner_path, new_content)
- signs_required = self.getSignsRequired(inner_path, new_content)
-
- if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
- signers_data = "%s:%s" % (signs_required, ",".join(valid_signers))
- if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]):
- raise VerifyError("Invalid signers_sign!")
-
- if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
- raise VerifyError("Invalid cert!")
-
- valid_signs = []
- for address in valid_signers:
- if address in signs:
- result = CryptBitcoin.verify(sign_content, address, signs[address])
- if result:
- valid_signs.append(address)
- if len(valid_signs) >= signs_required:
- break # Break if we has enough signs
- if len(valid_signs) < signs_required:
- raise VerifyError("Valid signs: %s/%s, Valid Signers : %s" % (len(valid_signs), signs_required, valid_signs))
- else:
- return self.verifyContent(inner_path, new_content)
- else: # Old style signing
- raise VerifyError("Invalid old-style sign")
-
- except Exception as err:
- self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err)))
- raise err
-
- else: # Check using sha512 hash
- file_info = self.getFileInfo(inner_path)
- if file_info:
- if CryptHash.sha512sum(file) != file_info.get("sha512", ""):
- raise VerifyError("Invalid hash")
-
- if file_info.get("size", 0) != file.tell():
- raise VerifyError(
- "File size does not match %s <> %s" %
- (inner_path, file.tell(), file_info.get("size", 0))
- )
-
- return True
-
- else: # File not in content.json
- raise VerifyError("File not in content.json")
+ try:
+ if inner_path.endswith("content.json"):
+ return self.verifyContentFile(inner_path, file, ignore_same)
+ else:
+ return self.verifyOrdinaryFile(inner_path, file, ignore_same)
+ except Exception as err:
+ self.log.info("%s: verify error: %s" % (inner_path, Debug.formatException(err)))
+ raise err
def optionalDelete(self, inner_path):
self.site.storage.delete(inner_path)
diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py
index c0903e84..ebbc6295 100644
--- a/src/Crypt/CryptConnection.py
+++ b/src/Crypt/CryptConnection.py
@@ -127,10 +127,6 @@ class CryptConnectionManager:
"/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Domain Validation Secure Server CA"
]
self.openssl_env['CN'] = random.choice(self.fakedomains)
- environ = os.environ
- environ['OPENSSL_CONF'] = self.openssl_env['OPENSSL_CONF']
- environ['RANDFILE'] = self.openssl_env['RANDFILE']
- environ['CN'] = self.openssl_env['CN']
if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem):
self.createSslContexts()
@@ -156,7 +152,7 @@ class CryptConnectionManager:
self.log.debug("Running: %s" % cmd)
proc = subprocess.Popen(
cmd, shell=True, stderr=subprocess.STDOUT,
- stdout=subprocess.PIPE, env=environ
+ stdout=subprocess.PIPE, env=self.openssl_env
)
back = proc.stdout.read().strip().decode(errors="replace").replace("\r", "")
proc.wait()
@@ -179,7 +175,7 @@ class CryptConnectionManager:
self.log.debug("Generating certificate key and signing request...")
proc = subprocess.Popen(
cmd, shell=True, stderr=subprocess.STDOUT,
- stdout=subprocess.PIPE, env=environ
+ stdout=subprocess.PIPE, env=self.openssl_env
)
back = proc.stdout.read().strip().decode(errors="replace").replace("\r", "")
proc.wait()
@@ -198,7 +194,7 @@ class CryptConnectionManager:
self.log.debug("Generating RSA cert...")
proc = subprocess.Popen(
cmd, shell=True, stderr=subprocess.STDOUT,
- stdout=subprocess.PIPE, env=environ
+ stdout=subprocess.PIPE, env=self.openssl_env
)
back = proc.stdout.read().strip().decode(errors="replace").replace("\r", "")
proc.wait()
diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py
index d100a3b8..c11185da 100644
--- a/src/Debug/DebugHook.py
+++ b/src/Debug/DebugHook.py
@@ -10,19 +10,33 @@ from Config import config
from . import Debug
last_error = None
+thread_shutdown = None
+
+thread_shutdown = None
+
+def shutdownThread():
+ import main
+ try:
+ if "file_server" in dir(main):
+ thread = gevent.spawn(main.file_server.stop)
+ thread.join(timeout=60)
+ if "ui_server" in dir(main):
+ thread = gevent.spawn(main.ui_server.stop)
+ thread.join(timeout=10)
+ except Exception as err:
+ print("Error in shutdown thread: %s" % err)
+ sys.exit(0)
+ else:
+ sys.exit(0)
def shutdown(reason="Unknown"):
+ global thread_shutdown
logging.info("Shutting down (reason: %s)..." % reason)
- import main
- if "file_server" in dir(main):
- try:
- gevent.spawn(main.file_server.stop)
- if "ui_server" in dir(main):
- gevent.spawn(main.ui_server.stop)
- except Exception as err:
- print("Proper shutdown error: %s" % err)
- sys.exit(0)
- else:
+ try:
+ if not thread_shutdown:
+ thread_shutdown = gevent.spawn(shutdownThread)
+ except Exception as err:
+ print("Proper shutdown error: %s" % err)
sys.exit(0)
# Store last error, ignore notify, allow manual error logging
diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py
index c082c378..d2dd1346 100644
--- a/src/File/FileRequest.py
+++ b/src/File/FileRequest.py
@@ -33,7 +33,7 @@ class FileRequest(object):
self.connection = connection
self.req_id = None
- self.sites = self.server.sites
+ self.sites = self.server.getSites()
self.log = server.log
self.responded = False # Responded to the request
@@ -109,35 +109,31 @@ class FileRequest(object):
return False
inner_path = params.get("inner_path", "")
+ current_content_modified = site.content_manager.contents.get(inner_path, {}).get("modified", 0)
+ body = params["body"]
+
if not inner_path.endswith("content.json"):
self.response({"error": "Only content.json update allowed"})
self.connection.badAction(5)
return
- current_content_modified = site.content_manager.contents.get(inner_path, {}).get("modified", 0)
should_validate_content = True
if "modified" in params and params["modified"] <= current_content_modified:
should_validate_content = False
valid = None # Same or earlier content as we have
-
- body = params["body"]
- if not body: # No body sent, we have to download it first
+ elif not body: # No body sent, we have to download it first
site.log.debug("Missing body from update for file %s, downloading ..." % inner_path)
peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer
try:
body = peer.getFile(site.address, inner_path).read()
except Exception as err:
site.log.debug("Can't download updated file %s: %s" % (inner_path, err))
- self.response({"error": "Invalid File update: Failed to download updated file content"})
+ self.response({"error": "File invalid update: Can't download updaed file"})
self.connection.badAction(5)
return
if should_validate_content:
try:
- if type(body) is str:
- body = body.encode()
- # elif type(body) is list:
- # content = json.loads(bytes(list).decode())
content = json.loads(body.decode())
except Exception as err:
site.log.debug("Update for %s is invalid JSON: %s" % (inner_path, err))
@@ -165,19 +161,21 @@ class FileRequest(object):
site.onFileDone(inner_path) # Trigger filedone
- # Download every changed file from peer
- peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer
- # On complete publish to other peers
- diffs = params.get("diffs", {})
- site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs, limit=6), "publish_%s" % inner_path)
+ if inner_path.endswith("content.json"): # Download every changed file from peer
+ peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer
+ # On complete publish to other peers
+ diffs = params.get("diffs", {})
+ site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs, limit=6), "publish_%s" % inner_path)
- # Load new content file and download changed files in new thread
- def downloader():
- site.downloadContent(inner_path, peer=peer, diffs=params.get("diffs", {}))
+ # Load new content file and download changed files in new thread
+ def downloader():
+ site.downloadContent(inner_path, peer=peer, diffs=params.get("diffs", {}))
+ del self.server.files_parsing[file_uri]
+
+ gevent.spawn(downloader)
+ else:
del self.server.files_parsing[file_uri]
- gevent.spawn(downloader)
-
self.response({"ok": "Thanks, file %s updated!" % inner_path})
self.connection.goodAction()
@@ -378,7 +376,7 @@ class FileRequest(object):
for hash_id, peers in found.items():
for peer in peers:
- ip_type = helper.getIpType(peer.ip)
+ ip_type = self.server.getIpType(peer.ip)
if len(back[ip_type][hash_id]) < 20:
back[ip_type][hash_id].append(peer.packMyAddress())
return back
@@ -432,7 +430,7 @@ class FileRequest(object):
# Check requested port of the other peer
def actionCheckport(self, params):
- if helper.getIpType(self.connection.ip) == "ipv6":
+ if self.server.getIpType(self.connection.ip) == "ipv6":
sock_address = (self.connection.ip, params["port"], 0, 0)
else:
sock_address = (self.connection.ip, params["port"])
diff --git a/src/File/FileServer.py b/src/File/FileServer.py
index b7a942fc..7114849b 100644
--- a/src/File/FileServer.py
+++ b/src/File/FileServer.py
@@ -3,6 +3,7 @@ import time
import random
import socket
import sys
+import weakref
import gevent
import gevent.pool
@@ -18,6 +19,13 @@ from Connection import ConnectionServer
from Plugin import PluginManager
from Debug import Debug
+log = logging.getLogger("FileServer")
+
+class FakeThread(object):
+ def __init__(self):
+ pass
+ def ready(self):
+ return False
@PluginManager.acceptPlugins
class FileServer(ConnectionServer):
@@ -25,12 +33,31 @@ class FileServer(ConnectionServer):
def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port, ip_type=config.fileserver_ip_type):
self.site_manager = SiteManager.site_manager
self.portchecker = PeerPortchecker.PeerPortchecker(self)
- self.log = logging.getLogger("FileServer")
self.ip_type = ip_type
self.ip_external_list = []
+ # This is wrong:
+ # self.log = logging.getLogger("FileServer")
+ # The value of self.log will be overwritten in ConnectionServer.__init__()
+
+ self.recheck_port = True
+
+ self.active_mode_thread_pool = gevent.pool.Pool(None)
+ self.site_pool = gevent.pool.Pool(None)
+
+ self.update_pool = gevent.pool.Pool(10)
+ self.update_start_time = 0
+ self.update_sites_task_next_nr = 1
+
+ self.update_threads = weakref.WeakValueDictionary()
+
+ self.passive_mode = None
+ self.active_mode = None
+ self.active_mode_threads = {}
+
+
self.supported_ip_types = ["ipv4"] # Outgoing ip_type support
- if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported():
+ if self.getIpType(ip) == "ipv6" or self.isIpv6Supported():
self.supported_ip_types.append("ipv6")
if ip_type == "ipv6" or (ip_type == "dual" and "ipv6" in self.supported_ip_types):
@@ -52,33 +79,50 @@ class FileServer(ConnectionServer):
config.arguments.fileserver_port = port
ConnectionServer.__init__(self, ip, port, self.handleRequest)
- self.log.debug("Supported IP types: %s" % self.supported_ip_types)
+ log.debug("Supported IP types: %s" % self.supported_ip_types)
+
+ self.managed_pools["active_mode_thread"] = self.active_mode_thread_pool
+ self.managed_pools["update"] = self.update_pool
+ self.managed_pools["site"] = self.site_pool
if ip_type == "dual" and ip == "::":
# Also bind to ipv4 addres in dual mode
try:
- self.log.debug("Binding proxy to %s:%s" % ("::", self.port))
+ log.debug("Binding proxy to %s:%s" % ("::", self.port))
self.stream_server_proxy = StreamServer(
("0.0.0.0", self.port), self.handleIncomingConnection, spawn=self.pool, backlog=100
)
except Exception as err:
- self.log.info("StreamServer proxy create error: %s" % Debug.formatException(err))
+ log.info("StreamServer proxy create error: %s" % Debug.formatException(err))
self.port_opened = {}
- self.sites = self.site_manager.sites
self.last_request = time.time()
self.files_parsing = {}
self.ui_server = None
+ def getSites(self):
+ sites = self.site_manager.list()
+ # We need to keep self.sites for the backward compatibility with plugins.
+ # Never. Ever. Use it.
+ # TODO: fix plugins
+ self.sites = sites
+ return sites
+
+ def getSite(self, address):
+ return self.getSites().get(address, None)
+
+ def getSiteAddresses(self):
+ # Avoid saving the site list on the stack, since a site may be deleted
+ # from the original list while iterating.
+ # Use the list of addresses instead.
+ return [
+ site.address for site in
+ sorted(list(self.getSites().values()), key=lambda site: site.settings.get("modified", 0), reverse=True)
+ ]
+
def getRandomPort(self, ip, port_range_from, port_range_to):
- """Generates Random Port from given range
- Args:
- ip: IP Address
- port_range_from: From Range
- port_range_to: to Range
- """
- self.log.info("Getting random port in range %s-%s..." % (port_range_from, port_range_to))
+ log.info("Getting random port in range %s-%s..." % (port_range_from, port_range_to))
tried = []
for bind_retry in range(100):
port = random.randint(port_range_from, port_range_to)
@@ -90,14 +134,14 @@ class FileServer(ConnectionServer):
sock.bind((ip, port))
success = True
except Exception as err:
- self.log.warning("Error binding to port %s: %s" % (port, err))
+ log.warning("Error binding to port %s: %s" % (port, err))
success = False
sock.close()
if success:
- self.log.info("Found unused random port: %s" % port)
+ log.info("Found unused random port: %s" % port)
return port
else:
- time.sleep(0.1)
+ self.sleep(0.1)
return False
def isIpv6Supported(self):
@@ -110,16 +154,16 @@ class FileServer(ConnectionServer):
sock.connect((ipv6_testip, 80))
local_ipv6 = sock.getsockname()[0]
if local_ipv6 == "::1":
- self.log.debug("IPv6 not supported, no local IPv6 address")
+ log.debug("IPv6 not supported, no local IPv6 address")
return False
else:
- self.log.debug("IPv6 supported on IP %s" % local_ipv6)
+ log.debug("IPv6 supported on IP %s" % local_ipv6)
return True
except socket.error as err:
- self.log.warning("IPv6 not supported: %s" % err)
+ log.warning("IPv6 not supported: %s" % err)
return False
except Exception as err:
- self.log.error("IPv6 check error: %s" % err)
+ log.error("IPv6 check error: %s" % err)
return False
def listenProxy(self):
@@ -127,29 +171,34 @@ class FileServer(ConnectionServer):
self.stream_server_proxy.serve_forever()
except Exception as err:
if err.errno == 98: # Address already in use error
- self.log.debug("StreamServer proxy listen error: %s" % err)
+ log.debug("StreamServer proxy listen error: %s" % err)
else:
- self.log.info("StreamServer proxy listen error: %s" % err)
+ log.info("StreamServer proxy listen error: %s" % err)
# Handle request to fileserver
def handleRequest(self, connection, message):
if config.verbose:
if "params" in message:
- self.log.debug(
+ log.debug(
"FileRequest: %s %s %s %s" %
(str(connection), message["cmd"], message["params"].get("site"), message["params"].get("inner_path"))
)
else:
- self.log.debug("FileRequest: %s %s" % (str(connection), message["cmd"]))
+ log.debug("FileRequest: %s %s" % (str(connection), message["cmd"]))
req = FileRequest(self, connection)
req.route(message["cmd"], message.get("req_id"), message.get("params"))
- if not self.has_internet and not connection.is_private_ip:
- self.has_internet = True
- self.onInternetOnline()
+ if not connection.is_private_ip:
+ self.setInternetStatus(True)
def onInternetOnline(self):
- self.log.info("Internet online")
- gevent.spawn(self.checkSites, check_files=False, force_port_check=True)
+ log.info("Internet online")
+ invalid_interval=(
+ self.internet_offline_since - self.internet_outage_threshold - random.randint(60 * 5, 60 * 10),
+ time.time()
+ )
+ self.invalidateUpdateTime(invalid_interval)
+ self.recheck_port = True
+ self.spawn(self.updateSites)
# Reload the FileRequest class to prevent restarts in debug mode
def reload(self):
@@ -158,8 +207,8 @@ class FileServer(ConnectionServer):
FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest
def portCheck(self):
- if config.offline:
- self.log.info("Offline mode: port check disabled")
+ if self.isOfflineMode():
+ log.info("Offline mode: port check disabled")
res = {"ipv4": None, "ipv6": None}
self.port_opened = res
return res
@@ -168,14 +217,14 @@ class FileServer(ConnectionServer):
for ip_external in config.ip_external:
SiteManager.peer_blacklist.append((ip_external, self.port)) # Add myself to peer blacklist
- ip_external_types = set([helper.getIpType(ip) for ip in config.ip_external])
+ ip_external_types = set([self.getIpType(ip) for ip in config.ip_external])
res = {
"ipv4": "ipv4" in ip_external_types,
"ipv6": "ipv6" in ip_external_types
}
self.ip_external_list = config.ip_external
self.port_opened.update(res)
- self.log.info("Server port opened based on configuration ipv4: %s, ipv6: %s" % (res["ipv4"], res["ipv6"]))
+ log.info("Server port opened based on configuration ipv4: %s, ipv6: %s" % (res["ipv4"], res["ipv6"]))
return res
self.port_opened = {}
@@ -183,7 +232,7 @@ class FileServer(ConnectionServer):
self.ui_server.updateWebsocket()
if "ipv6" in self.supported_ip_types:
- res_ipv6_thread = gevent.spawn(self.portchecker.portCheck, self.port, "ipv6")
+ res_ipv6_thread = self.spawn(self.portchecker.portCheck, self.port, "ipv6")
else:
res_ipv6_thread = None
@@ -196,8 +245,8 @@ class FileServer(ConnectionServer):
res_ipv6 = {"ip": None, "opened": None}
else:
res_ipv6 = res_ipv6_thread.get()
- if res_ipv6["opened"] and not helper.getIpType(res_ipv6["ip"]) == "ipv6":
- self.log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"])
+ if res_ipv6["opened"] and not self.getIpType(res_ipv6["ip"]) == "ipv6":
+ log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"])
res_ipv6["opened"] = False
self.ip_external_list = []
@@ -206,7 +255,7 @@ class FileServer(ConnectionServer):
self.ip_external_list.append(res_ip["ip"])
SiteManager.peer_blacklist.append((res_ip["ip"], self.port))
- self.log.info("Server port opened ipv4: %s, ipv6: %s" % (res_ipv4["opened"], res_ipv6["opened"]))
+ log.info("Server port opened ipv4: %s, ipv6: %s" % (res_ipv4["opened"], res_ipv6["opened"]))
res = {"ipv4": res_ipv4["opened"], "ipv6": res_ipv6["opened"]}
@@ -217,9 +266,9 @@ class FileServer(ConnectionServer):
for ip in interface_ips:
if not helper.isPrivateIp(ip) and ip not in self.ip_external_list:
self.ip_external_list.append(ip)
- res[helper.getIpType(ip)] = True # We have opened port if we have external ip
+ res[self.getIpType(ip)] = True # We have opened port if we have external ip
SiteManager.peer_blacklist.append((ip, self.port))
- self.log.debug("External ip found on interfaces: %s" % ip)
+ log.debug("External ip found on interfaces: %s" % ip)
self.port_opened.update(res)
@@ -228,131 +277,381 @@ class FileServer(ConnectionServer):
return res
- # Check site file integrity
- def checkSite(self, site, check_files=False):
- if site.isServing():
- site.announce(mode="startup") # Announce site to tracker
- site.update(check_files=check_files) # Update site's content.json and download changed files
- site.sendMyHashfield()
- site.updateHashfield()
-
- # Check sites integrity
- @util.Noparallel()
- def checkSites(self, check_files=False, force_port_check=False):
- self.log.debug("Checking sites...")
- s = time.time()
- sites_checking = False
- if not self.port_opened or force_port_check: # Test and open port if not tested yet
- if len(self.sites) <= 2: # Don't wait port opening on first startup
- sites_checking = True
- for address, site in list(self.sites.items()):
- gevent.spawn(self.checkSite, site, check_files)
-
+ @util.Noparallel(queue=True)
+ def recheckPort(self):
+ if self.recheck_port:
self.portCheck()
+ self.recheck_port = False
- if not self.port_opened["ipv4"]:
- self.tor_manager.startOnions()
+ # Returns False if Internet is immediately available
+ # Returns True if we've spent some time waiting for Internet
+ # Returns None if FileServer is stopping or the Offline mode is enabled
+ @util.Noparallel()
+ def waitForInternetOnline(self):
+ if self.isOfflineMode() or self.stopping:
+ return None
- if not sites_checking:
- check_pool = gevent.pool.Pool(5)
- # Check sites integrity
- for site in sorted(list(self.sites.values()), key=lambda site: site.settings.get("modified", 0), reverse=True):
- if not site.isServing():
- continue
- check_thread = check_pool.spawn(self.checkSite, site, check_files) # Check in new thread
- time.sleep(2)
- if site.settings.get("modified", 0) < time.time() - 60 * 60 * 24: # Not so active site, wait some sec to finish
- check_thread.join(timeout=5)
- self.log.debug("Checksites done in %.3fs" % (time.time() - s))
+ if self.isInternetOnline():
+ return False
+
+ while not self.isInternetOnline():
+ self.sleep(30)
+ if self.isOfflineMode() or self.stopping:
+ return None
+ if self.isInternetOnline():
+ break
+ if len(self.update_pool) == 0:
+ log.info("Internet connection seems to be broken. Running an update for a random site to check if we are able to connect to any peer.")
+ thread = self.thread_pool.spawn(self.updateRandomSite)
+ thread.join()
+
+ self.recheckPort()
+ return True
+
+ def updateRandomSite(self, site_addresses=None, force=False):
+ if not site_addresses:
+ site_addresses = self.getSiteAddresses()
+
+ site_addresses = random.sample(site_addresses, 1)
+ if len(site_addresses) < 1:
+ return
+
+ address = site_addresses[0]
+ site = self.getSite(address)
+
+ if not site:
+ return
+
+ log.info("Randomly chosen site: %s", site.address_short)
+
+ self.spawnUpdateSite(site).join()
+
+ def updateSite(self, site, check_files=False, verify_files=False):
+ if not site:
+ return
+ if verify_files:
+ mode = 'verify'
+ elif check_files:
+ mode = 'check'
+ else:
+ mode = 'update'
+ log.info("running <%s> for %s" % (mode, site.address_short))
+ site.update2(check_files=check_files, verify_files=verify_files)
+
+ def spawnUpdateSite(self, site, check_files=False, verify_files=False):
+ fake_thread = FakeThread()
+ self.update_threads[site.address] = fake_thread
+ thread = self.update_pool.spawn(self.updateSite, site,
+ check_files=check_files, verify_files=verify_files)
+ self.update_threads[site.address] = thread
+ return thread
+
+ def lookupInUpdatePool(self, site_address):
+ thread = self.update_threads.get(site_address, None)
+ if not thread or thread.ready():
+ return None
+ return thread
+
+ def siteIsInUpdatePool(self, site_address):
+ return self.lookupInUpdatePool(site_address) is not None
+
+ def invalidateUpdateTime(self, invalid_interval):
+ for address in self.getSiteAddresses():
+ site = self.getSite(address)
+ if site:
+ site.invalidateUpdateTime(invalid_interval)
+
+ def isSiteUpdateTimeValid(self, site_address):
+ site = self.getSite(site_address)
+ if not site:
+ return False
+ return site.isUpdateTimeValid()
+
+ def updateSites(self):
+ task_nr = self.update_sites_task_next_nr
+ self.update_sites_task_next_nr += 1
+
+ task_description = "updateSites [#%d]" % task_nr
+ log.info("%s: started", task_description)
+
+ # Don't wait port opening on first startup. Do the instant check now.
+ if len(self.getSites()) <= 2:
+ for address, site in list(self.getSites().items()):
+ self.updateSite(site, check_files=True)
+
+ self.recheckPort()
+
+ all_site_addresses = self.getSiteAddresses()
+ site_addresses = [
+ address for address in all_site_addresses
+ if not self.isSiteUpdateTimeValid(address)
+ ]
+
+ log.info("%s: chosen %d sites (of %d)", task_description, len(site_addresses), len(all_site_addresses))
+
+ sites_processed = 0
+ sites_skipped = 0
+ start_time = time.time()
+ self.update_start_time = start_time
+ progress_print_time = time.time()
+
+ # Check sites integrity
+ for site_address in site_addresses:
+ site = None
+ self.sleep(1)
+ self.waitForInternetOnline()
+
+ while self.isActiveMode() and self.shouldThrottleNewConnections():
+ self.sleep(1)
+
+ if not self.isActiveMode():
+ break
+
+ site = self.getSite(site_address)
+ if not site or site.isUpdateTimeValid() or self.siteIsInUpdatePool(site_address):
+ sites_skipped += 1
+ continue
+
+ sites_processed += 1
+ thread = self.spawnUpdateSite(site)
+
+ if not self.isActiveMode():
+ break
+
+ if time.time() - progress_print_time > 60:
+ progress_print_time = time.time()
+ time_spent = time.time() - start_time
+ time_per_site = time_spent / float(sites_processed)
+ sites_left = len(site_addresses) - sites_processed
+ time_left = time_per_site * sites_left
+ log.info("%s: DONE: %d sites in %.2fs (%.2fs per site); SKIPPED: %d sites; LEFT: %d sites in %.2fs",
+ task_description,
+ sites_processed,
+ time_spent,
+ time_per_site,
+ sites_skipped,
+ sites_left,
+ time_left
+ )
+
+ if not self.isActiveMode():
+ log.info("%s: stopped", task_description)
+ else:
+ log.info("%s: finished in %.2fs", task_description, time.time() - start_time)
+
+ def peekSiteForVerification(self):
+ check_files_interval = 60 * 60 * 24
+ verify_files_interval = 60 * 60 * 24 * 10
+ site_addresses = self.getSiteAddresses()
+ random.shuffle(site_addresses)
+ for site_address in site_addresses:
+ site = self.getSite(site_address)
+ if not site:
+ continue
+ mode = site.isFileVerificationExpired(check_files_interval, verify_files_interval)
+ if mode:
+ return (site_address, mode)
+ return (None, None)
+
+
+ def sitesVerificationThread(self):
+ log.info("sitesVerificationThread started")
+ short_timeout = 20
+ long_timeout = 120
+
+ self.sleep(long_timeout)
+
+ while self.isActiveMode():
+ site = None
+ self.sleep(short_timeout)
+ self.waitForInternetOnline()
+
+ while self.isActiveMode() and self.shouldThrottleNewConnections():
+ self.sleep(1)
+
+ if not self.isActiveMode():
+ break
+
+ site_address, mode = self.peekSiteForVerification()
+ if not site_address:
+ self.sleep(long_timeout)
+ continue
+
+ while self.siteIsInUpdatePool(site_address) and self.isActiveMode():
+ self.sleep(1)
+
+ if not self.isActiveMode():
+ break
+
+ site = self.getSite(site_address)
+ if not site:
+ continue
+
+ if mode == "verify":
+ check_files = False
+ verify_files = True
+ elif mode == "check":
+ check_files = True
+ verify_files = False
+ else:
+ continue
+
+ thread = self.spawnUpdateSite(site,
+ check_files=check_files, verify_files=verify_files)
+
+ log.info("sitesVerificationThread stopped")
+
+ def sitesMaintenanceThread(self, mode="full"):
+ log.info("sitesMaintenanceThread(%s) started" % mode)
- def cleanupSites(self):
- import gc
startup = True
- time.sleep(5 * 60) # Sites already cleaned up on startup
- peers_protected = set([])
- while 1:
- # Sites health care every 20 min
- self.log.debug(
- "Running site cleanup, connections: %s, internet: %s, protected peers: %s" %
- (len(self.connections), self.has_internet, len(peers_protected))
+
+ short_timeout = 2
+ min_long_timeout = 10
+ max_long_timeout = 60 * 10
+ long_timeout = min_long_timeout
+ short_cycle_time_limit = 60 * 2
+
+ while self.isActiveMode():
+ self.sleep(long_timeout)
+
+ while self.isActiveMode() and self.shouldThrottleNewConnections():
+ self.sleep(1)
+
+ if not self.isActiveMode():
+ break
+
+ start_time = time.time()
+
+ log.debug(
+ "Starting <%s> maintenance cycle: connections=%s, internet=%s",
+ mode,
+ len(self.connections), self.isInternetOnline()
+ )
+ start_time = time.time()
+
+ site_addresses = self.getSiteAddresses()
+
+ sites_processed = 0
+
+ for site_address in site_addresses:
+ if not self.isActiveMode():
+ break
+
+ site = self.getSite(site_address)
+ if not site:
+ continue
+
+ log.debug("Running maintenance for site: %s", site.address_short)
+
+ done = site.runPeriodicMaintenance(startup=startup)
+ site = None
+ if done:
+ sites_processed += 1
+ self.sleep(short_timeout)
+
+ # If we host hundreds of sites, the full maintenance cycle may take very
+ # long time, especially on startup ( > 1 hour).
+ # This means we are not able to run the maintenance procedure for active
+ # sites frequently enough using just a single maintenance thread.
+ # So we run 2 maintenance threads:
+ # * One running full cycles.
+ # * And one running short cycles for the most active sites.
+ # When the short cycle runs out of the time limit, it restarts
+ # from the beginning of the site list.
+ if mode == "short" and time.time() - start_time > short_cycle_time_limit:
+ break
+
+ log.debug("<%s> maintenance cycle finished in %.2fs. Total sites: %d. Processed sites: %d. Timeout: %d",
+ mode,
+ time.time() - start_time,
+ len(site_addresses),
+ sites_processed,
+ long_timeout
)
- for address, site in list(self.sites.items()):
- if not site.isServing():
- continue
+ if sites_processed:
+ long_timeout = max(int(long_timeout / 2), min_long_timeout)
+ else:
+ long_timeout = min(long_timeout + 1, max_long_timeout)
- if not startup:
- site.cleanupPeers(peers_protected)
-
- time.sleep(1) # Prevent too quick request
-
- peers_protected = set([])
- for address, site in list(self.sites.items()):
- if not site.isServing():
- continue
-
- if site.peers:
- with gevent.Timeout(10, exception=False):
- site.announcer.announcePex()
-
- # Last check modification failed
- if site.content_updated is False:
- site.update()
- elif site.bad_files:
- site.retryBadFiles()
-
- if time.time() - site.settings.get("modified", 0) < 60 * 60 * 24 * 7:
- # Keep active connections if site has been modified witin 7 days
- connected_num = site.needConnections(check_site_on_reconnect=True)
-
- if connected_num < config.connected_limit: # This site has small amount of peers, protect them from closing
- peers_protected.update([peer.key for peer in site.getConnectedPeers()])
-
- time.sleep(1) # Prevent too quick request
-
- site = None
- gc.collect() # Implicit garbage collection
+ site_addresses = None
startup = False
- time.sleep(60 * 20)
+ log.info("sitesMaintenanceThread(%s) stopped" % mode)
- def announceSite(self, site):
- site.announce(mode="update", pex=False)
- active_site = time.time() - site.settings.get("modified", 0) < 24 * 60 * 60
- if site.settings["own"] or active_site:
- # Check connections more frequently on own and active sites to speed-up first connections
- site.needConnections(check_site_on_reconnect=True)
- site.sendMyHashfield(3)
- site.updateHashfield(3)
+ def keepAliveThread(self):
+ # This thread is mostly useless on a system under load, since it never does
+ # any works, if we have active traffic.
+ #
+ # We should initiate some network activity to detect the Internet outage
+ # and avoid false positives. We normally have some network activity
+ # initiated by various parts on the application as well as network peers.
+ # So it's not a problem.
+ #
+ # However, if it actually happens that we have no network traffic for
+ # some time (say, we host just a couple of inactive sites, and no peers
+ # are interested in connecting to them), we initiate some traffic by
+ # performing the update for a random site. It's way better than just
+ # silly pinging a random peer for no profit.
+ log.info("keepAliveThread started")
+ while self.isActiveMode():
+ self.waitForInternetOnline()
- # Announce sites every 20 min
- def announceSites(self):
- time.sleep(5 * 60) # Sites already announced on startup
- while 1:
+ threshold = self.internet_outage_threshold / 2.0
+
+ self.sleep(threshold / 2.0)
+
+ while self.isActiveMode() and self.shouldThrottleNewConnections():
+ self.sleep(1)
+
+ if not self.isActiveMode():
+ break
+
+ last_activity_time = max(
+ self.last_successful_internet_activity_time,
+ self.last_outgoing_internet_activity_time)
+ now = time.time()
+ if not len(self.getSites()):
+ continue
+ if last_activity_time > now - threshold:
+ continue
+ if len(self.update_pool) != 0:
+ continue
+
+ log.info("No network activity for %.2fs. Running an update for a random site.",
+ now - last_activity_time
+ )
+ self.update_pool.spawn(self.updateRandomSite, force=True)
+ log.info("keepAliveThread stopped")
+
+ # Periodic reloading of tracker files
+ def reloadTrackerFilesThread(self):
+ # TODO:
+ # This should probably be more sophisticated.
+ # We should check if the files have actually changed,
+ # and do it more often.
+ log.info("reloadTrackerFilesThread started")
+ interval = 60 * 10
+ while self.isActiveMode():
+ self.sleep(interval)
+ if not self.isActiveMode():
+ break
config.loadTrackersFile()
- s = time.time()
- for address, site in list(self.sites.items()):
- if not site.isServing():
- continue
- gevent.spawn(self.announceSite, site).join(timeout=10)
- time.sleep(1)
- taken = time.time() - s
-
- # Query all trackers one-by-one in 20 minutes evenly distributed
- sleep = max(0, 60 * 20 / len(config.trackers) - taken)
-
- self.log.debug("Site announce tracker done in %.3fs, sleeping for %.3fs..." % (taken, sleep))
- time.sleep(sleep)
+ log.info("reloadTrackerFilesThread stopped")
# Detects if computer back from wakeup
- def wakeupWatcher(self):
+ def wakeupWatcherThread(self):
+ log.info("wakeupWatcherThread started")
last_time = time.time()
last_my_ips = socket.gethostbyname_ex('')[2]
- while 1:
- time.sleep(30)
+ while self.isActiveMode():
+ self.sleep(30)
+ if not self.isActiveMode():
+ break
is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3
if is_time_changed:
# If taken more than 3 minute then the computer was in sleep mode
- self.log.info(
+ log.info(
"Wakeup detected: time warp from %0.f to %0.f (%0.f sleep seconds), acting like startup..." %
(last_time, time.time(), time.time() - last_time)
)
@@ -360,50 +659,130 @@ class FileServer(ConnectionServer):
my_ips = socket.gethostbyname_ex('')[2]
is_ip_changed = my_ips != last_my_ips
if is_ip_changed:
- self.log.info("IP change detected from %s to %s" % (last_my_ips, my_ips))
+ log.info("IP change detected from %s to %s" % (last_my_ips, my_ips))
if is_time_changed or is_ip_changed:
- self.checkSites(check_files=False, force_port_check=True)
+ invalid_interval=(
+ last_time - self.internet_outage_threshold - random.randint(60 * 5, 60 * 10),
+ time.time()
+ )
+ self.invalidateUpdateTime(invalid_interval)
+ self.recheck_port = True
+ self.spawn(self.updateSites)
last_time = time.time()
last_my_ips = my_ips
+ log.info("wakeupWatcherThread stopped")
+
+ def setOfflineMode(self, offline_mode):
+ ConnectionServer.setOfflineMode(self, offline_mode)
+ self.setupActiveMode()
+
+ def setPassiveMode(self, passive_mode):
+ if self.passive_mode == passive_mode:
+ return
+ self.passive_mode = passive_mode
+ if self.passive_mode:
+ log.info("passive mode is ON");
+ else:
+ log.info("passive mode is OFF");
+ self.setupActiveMode()
+
+ def isPassiveMode(self):
+ return self.passive_mode
+
+ def setupActiveMode(self):
+ active_mode = (not self.passive_mode) and (not self.isOfflineMode())
+ if self.active_mode == active_mode:
+ return
+ self.active_mode = active_mode
+ if self.active_mode:
+ log.info("active mode is ON");
+ self.enterActiveMode();
+ else:
+ log.info("active mode is OFF");
+ self.leaveActiveMode();
+
+ def killActiveModeThreads(self):
+ for key, thread in list(self.active_mode_threads.items()):
+ if thread:
+ if not thread.ready():
+ log.info("killing %s" % key)
+ gevent.kill(thread)
+ del self.active_mode_threads[key]
+
+ def leaveActiveMode(self):
+ pass
+
+ def enterActiveMode(self):
+ self.killActiveModeThreads()
+ x = self.active_mode_threads
+ p = self.active_mode_thread_pool
+ x["thread_keep_alive"] = p.spawn(self.keepAliveThread)
+ x["thread_wakeup_watcher"] = p.spawn(self.wakeupWatcherThread)
+ x["thread_sites_verification"] = p.spawn(self.sitesVerificationThread)
+ x["thread_reload_tracker_files"] = p.spawn(self.reloadTrackerFilesThread)
+ x["thread_sites_maintenance_full"] = p.spawn(self.sitesMaintenanceThread, mode="full")
+ x["thread_sites_maintenance_short"] = p.spawn(self.sitesMaintenanceThread, mode="short")
+ x["thread_initial_site_updater"] = p.spawn(self.updateSites)
+
+ # Returns True, if an active mode thread should keep going,
+ # i.e active mode is enabled and the server not going to shutdown
+ def isActiveMode(self):
+ self.setupActiveMode()
+ if not self.active_mode:
+ return False
+ if not self.running:
+ return False
+ if self.stopping:
+ return False
+ return True
# Bind and start serving sites
- def start(self, check_sites=True):
+ # If passive_mode is False, FileServer starts the full-featured file serving:
+ # * Checks for updates at startup.
+ # * Checks site's integrity.
+ # * Runs periodic update checks.
+ # * Watches for internet being up or down and for computer to wake up and runs update checks.
+ # If passive_mode is True, all the mentioned activity is disabled.
+ def start(self, passive_mode=False, check_sites=None, check_connections=True):
+
+ # Backward compatibility for a misnamed argument:
+ if check_sites is not None:
+ passive_mode = not check_sites
+
if self.stopping:
return False
- ConnectionServer.start(self)
+ ConnectionServer.start(self, check_connections=check_connections)
try:
self.stream_server.start()
except Exception as err:
- self.log.error("Error listening on: %s:%s: %s" % (self.ip, self.port, err))
+ log.error("Error listening on: %s:%s: %s" % (self.ip, self.port, err))
- self.sites = self.site_manager.list()
if config.debug:
# Auto reload FileRequest on change
from Debug import DebugReloader
DebugReloader.watcher.addCallback(self.reload)
- if check_sites: # Open port, Update sites, Check files integrity
- gevent.spawn(self.checkSites)
+ # XXX: for initializing self.sites
+ # Remove this line when self.sites gets completely unused
+ self.getSites()
- thread_announce_sites = gevent.spawn(self.announceSites)
- thread_cleanup_sites = gevent.spawn(self.cleanupSites)
- thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher)
+ self.setPassiveMode(passive_mode)
ConnectionServer.listen(self)
- self.log.debug("Stopped.")
+ log.info("Stopped.")
- def stop(self):
+ def stop(self, ui_websocket=None):
if self.running and self.portchecker.upnp_port_opened:
- self.log.debug('Closing port %d' % self.port)
+ log.debug('Closing port %d' % self.port)
try:
self.portchecker.portClose(self.port)
- self.log.info('Closed port via upnp.')
+ log.info('Closed port via upnp.')
except Exception as err:
- self.log.info("Failed at attempt to use upnp to close port: %s" % err)
+ log.info("Failed at attempt to use upnp to close port: %s" % err)
- return ConnectionServer.stop(self)
+ return ConnectionServer.stop(self, ui_websocket=ui_websocket)
diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py
index 03cc1f47..43c2932f 100644
--- a/src/Peer/Peer.py
+++ b/src/Peer/Peer.py
@@ -20,51 +20,135 @@ if config.use_tempfiles:
# Communicate remote peers
@PluginManager.acceptPlugins
class Peer(object):
- __slots__ = (
- "ip", "port", "site", "key", "connection", "connection_server", "time_found", "time_response", "time_hashfield",
- "time_added", "has_hashfield", "is_tracker_connection", "time_my_hashfield_sent", "last_ping", "reputation",
- "last_content_json_update", "hashfield", "connection_error", "hash_failed", "download_bytes", "download_time"
- )
-
def __init__(self, ip, port, site=None, connection_server=None):
self.ip = ip
self.port = port
self.site = site
self.key = "%s:%s" % (ip, port)
+ self.ip_type = None
+
+ self.removed = False
+
+ self.log_level = logging.DEBUG
+ self.connection_error_log_level = logging.DEBUG
+
self.connection = None
self.connection_server = connection_server
self.has_hashfield = False # Lazy hashfield object not created yet
self.time_hashfield = None # Last time peer's hashfiled downloaded
self.time_my_hashfield_sent = None # Last time my hashfield sent to peer
self.time_found = time.time() # Time of last found in the torrent tracker
- self.time_response = None # Time of last successful response from peer
+ self.time_response = 0 # Time of last successful response from peer
self.time_added = time.time()
self.last_ping = None # Last response time for ping
+ self.last_pex = 0 # Last query/response time for pex
self.is_tracker_connection = False # Tracker connection instead of normal peer
self.reputation = 0 # More likely to connect if larger
self.last_content_json_update = 0.0 # Modify date of last received content.json
+ self.protected = 0
+ self.reachable = None
self.connection_error = 0 # Series of connection error
self.hash_failed = 0 # Number of bad files from peer
self.download_bytes = 0 # Bytes downloaded
self.download_time = 0 # Time spent to download
+ self.protectedRequests = ["getFile", "streamFile", "update", "listModified"]
+
def __getattr__(self, key):
if key == "hashfield":
self.has_hashfield = True
self.hashfield = PeerHashfield()
return self.hashfield
else:
- return getattr(self, key)
+ # Raise appropriately formatted attribute error
+ return object.__getattribute__(self, key)
+
+ def log(self, text, log_level = None):
+ if log_level is None:
+ log_level = self.log_level
+ if log_level <= logging.DEBUG:
+ if not config.verbose:
+ return # Only log if we are in debug mode
+
+ logger = None
- def log(self, text):
- if not config.verbose:
- return # Only log if we are in debug mode
if self.site:
- self.site.log.debug("%s:%s %s" % (self.ip, self.port, text))
+ logger = self.site.log
else:
- logging.debug("%s:%s %s" % (self.ip, self.port, text))
+ logger = logging.getLogger()
+
+ logger.log(log_level, "%s:%s %s" % (self.ip, self.port, text))
+
+ # Protect connection from being closed by site.cleanupPeers()
+ def markProtected(self, interval=60*2):
+ self.protected = max(self.protected, time.time() + interval)
+
+ def isProtected(self):
+ if self.protected > 0:
+ if self.protected < time.time():
+ self.protected = 0
+ return self.protected > 0
+
+ def isTtlExpired(self, ttl):
+ last_activity = max(self.time_found, self.time_response)
+ return (time.time() - last_activity) > ttl
+
+ # Since 0.8.0
+ def isConnected(self):
+ if self.connection and not self.connection.connected:
+ self.connection = None
+ return self.connection and self.connection.connected
+
+ # Peer proved to to be connectable recently
+ # Since 0.8.0
+ def isConnectable(self):
+ if self.connection_error >= 1: # The last connection attempt failed
+ return False
+ if time.time() - self.time_response > 60 * 60 * 2: # Last successful response more than 2 hours ago
+ return False
+ return self.isReachable()
+
+ # Since 0.8.0
+ def isReachable(self):
+ if self.reachable is None:
+ self.updateCachedState()
+ return self.reachable
+
+ # Since 0.8.0
+ def getIpType(self):
+ if not self.ip_type:
+ self.updateCachedState()
+ return self.ip_type
+
+ # We cache some ConnectionServer-related state for better performance.
+ # This kind of state currently doesn't change during a program session,
+ # and it's safe to read and cache it just once. But future versions
+ # may bring more pieces of dynamic configuration. So we update the state
+ # on each peer.found().
+ def updateCachedState(self):
+ connection_server = self.getConnectionServer()
+ if not self.port or self.port == 1: # Port 1 considered as "no open port"
+ self.reachable = False
+ else:
+ self.reachable = connection_server.isIpReachable(self.ip)
+ self.ip_type = connection_server.getIpType(self.ip)
+
+
+ # FIXME:
+ # This should probably be changed.
+ # When creating a peer object, the caller must provide either `connection_server`,
+ # or `site`, so Peer object is able to use `site.connection_server`.
+ def getConnectionServer(self):
+ if self.connection_server:
+ connection_server = self.connection_server
+ elif self.site:
+ connection_server = self.site.connection_server
+ else:
+ import main
+ connection_server = main.file_server
+ return connection_server
# Connect to host
def connect(self, connection=None):
@@ -87,29 +171,30 @@ class Peer(object):
self.connection = None
try:
- if self.connection_server:
- connection_server = self.connection_server
- elif self.site:
- connection_server = self.site.connection_server
- else:
- import main
- connection_server = main.file_server
+ connection_server = self.getConnectionServer()
self.connection = connection_server.getConnection(self.ip, self.port, site=self.site, is_tracker_connection=self.is_tracker_connection)
- self.reputation += 1
- self.connection.sites += 1
+ if self.connection and self.connection.connected:
+ self.reputation += 1
+ self.connection.sites += 1
except Exception as err:
self.onConnectionError("Getting connection error")
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
- (Debug.formatException(err), self.connection_error, self.hash_failed))
+ (Debug.formatException(err), self.connection_error, self.hash_failed),
+ log_level=self.connection_error_log_level)
self.connection = None
return self.connection
+ def disconnect(self, reason="Unknown"):
+ if self.connection:
+ self.connection.close(reason)
+ self.connection = None
+
# Check if we have connection to peer
def findConnection(self):
if self.connection and self.connection.connected: # We have connection to peer
return self.connection
else: # Try to find from other sites connections
- self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False, site=self.site)
+ self.connection = self.getConnectionServer().getConnection(self.ip, self.port, create=False, site=self.site)
if self.connection:
self.connection.sites += 1
return self.connection
@@ -143,9 +228,13 @@ class Peer(object):
if source in ("tracker", "local"):
self.site.peers_recent.appendleft(self)
self.time_found = time.time()
+ self.updateCachedState()
# Send a command to peer and return response value
def request(self, cmd, params={}, stream_to=None):
+ if self.removed:
+ return False
+
if not self.connection or self.connection.closed:
self.connect()
if not self.connection:
@@ -156,6 +245,8 @@ class Peer(object):
for retry in range(1, 4): # Retry 3 times
try:
+ if cmd in self.protectedRequests:
+ self.markProtected()
if not self.connection:
raise Exception("No connection found")
res = self.connection.request(cmd, params, stream_to)
@@ -188,6 +279,9 @@ class Peer(object):
# Get a file content from peer
def getFile(self, site, inner_path, file_size=None, pos_from=0, pos_to=None, streaming=False):
+ if self.removed:
+ return False
+
if file_size and file_size > 5 * 1024 * 1024:
max_read_size = 1024 * 1024
else:
@@ -241,11 +335,14 @@ class Peer(object):
return buff
# Send a ping request
- def ping(self):
+ def ping(self, timeout=10.0, tryes=3):
+ if self.removed:
+ return False
+
response_time = None
- for retry in range(1, 3): # Retry 3 times
+ for retry in range(1, tryes): # Retry 3 times
s = time.time()
- with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception
+ with gevent.Timeout(timeout, False):
res = self.request("ping")
if res and "body" in res and res["body"] == b"Pong!":
@@ -264,10 +361,18 @@ class Peer(object):
return response_time
# Request peer exchange from peer
- def pex(self, site=None, need_num=5):
+ def pex(self, site=None, need_num=5, request_interval=60*2):
+ if self.removed:
+ return False
+
if not site:
site = self.site # If no site defined request peers for this site
+ if self.last_pex + request_interval >= time.time():
+ return False
+
+ self.last_pex = time.time()
+
# give back 5 connectible peers
packed_peers = helper.packPeers(self.site.getConnectablePeers(5, allow_private=False))
request = {"site": site.address, "peers": packed_peers["ipv4"], "need": need_num}
@@ -276,6 +381,7 @@ class Peer(object):
if packed_peers["ipv6"]:
request["peers_ipv6"] = packed_peers["ipv6"]
res = self.request("pex", request)
+ self.last_pex = time.time()
if not res or "error" in res:
return False
added = 0
@@ -307,9 +413,14 @@ class Peer(object):
# List modified files since the date
# Return: {inner_path: modification date,...}
def listModified(self, since):
+ if self.removed:
+ return False
return self.request("listModified", {"since": since, "site": self.site.address})
def updateHashfield(self, force=False):
+ if self.removed:
+ return False
+
# Don't update hashfield again in 5 min
if self.time_hashfield and time.time() - self.time_hashfield < 5 * 60 and not force:
return False
@@ -325,6 +436,9 @@ class Peer(object):
# Find peers for hashids
# Return: {hash1: ["ip:port", "ip:port",...],...}
def findHashIds(self, hash_ids):
+ if self.removed:
+ return False
+
res = self.request("findHashIds", {"site": self.site.address, "hash_ids": hash_ids})
if not res or "error" in res or type(res) is not dict:
return False
@@ -368,6 +482,9 @@ class Peer(object):
return True
def publish(self, address, inner_path, body, modified, diffs=[]):
+ if self.removed:
+ return False
+
if len(body) > 10 * 1024 and self.connection and self.connection.handshake.get("rev", 0) >= 4095:
# To save bw we don't push big content.json to peers
body = b""
@@ -382,20 +499,22 @@ class Peer(object):
# Stop and remove from site
def remove(self, reason="Removing"):
- self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed))
- if self.site and self.key in self.site.peers:
- del(self.site.peers[self.key])
+ self.removed = True
+ self.log("Removing peer with reason: <%s>. Connection error: %s, Hash failed: %s" % (reason, self.connection_error, self.hash_failed))
+ if self.site:
+ self.site.deregisterPeer(self)
+ # No way: self.site = None
+ # We don't assign None to self.site here because it leads to random exceptions in various threads,
+ # that hold references to the peer and still believe it belongs to the site.
- if self.site and self in self.site.peers_recent:
- self.site.peers_recent.remove(self)
-
- if self.connection:
- self.connection.close(reason)
+ self.disconnect(reason)
# - EVENTS -
# On connection error
def onConnectionError(self, reason="Unknown"):
+ if not self.getConnectionServer().isInternetOnline():
+ return
self.connection_error += 1
if self.site and len(self.site.peers) > 200:
limit = 3
@@ -403,7 +522,7 @@ class Peer(object):
limit = 6
self.reputation -= 1
if self.connection_error >= limit: # Dead peer
- self.remove("Peer connection: %s" % reason)
+ self.remove("Connection error limit reached: %s. Provided message: %s" % (limit, reason))
# Done working with peer
def onWorkerDone(self):
diff --git a/src/Plugin/PluginManager.py b/src/Plugin/PluginManager.py
index 56540e60..dbafa98f 100644
--- a/src/Plugin/PluginManager.py
+++ b/src/Plugin/PluginManager.py
@@ -16,9 +16,7 @@ import plugins
class PluginManager:
def __init__(self):
self.log = logging.getLogger("PluginManager")
- self.path_plugins = None
- if plugins.__file__:
- self.path_plugins = os.path.dirname(os.path.abspath(plugins.__file__));
+ self.path_plugins = os.path.abspath(os.path.dirname(plugins.__file__))
self.path_installed_plugins = config.data_dir + "/__plugins__"
self.plugins = defaultdict(list) # Registered plugins (key: class name, value: list of plugins for class)
self.subclass_order = {} # Record the load order of the plugins, to keep it after reload
@@ -34,8 +32,7 @@ class PluginManager:
self.config.setdefault("builtin", {})
- if self.path_plugins:
- sys.path.append(os.path.join(os.getcwd(), self.path_plugins))
+ sys.path.append(os.path.join(os.getcwd(), self.path_plugins))
self.migratePlugins()
if config.debug: # Auto reload Plugins on file change
@@ -130,8 +127,6 @@ class PluginManager:
def loadPlugins(self):
all_loaded = True
s = time.time()
- if self.path_plugins is None:
- return
for plugin in self.listPlugins():
self.log.debug("Loading plugin: %s (%s)" % (plugin["name"], plugin["source"]))
if plugin["source"] != "builtin":
diff --git a/src/Site/Site.py b/src/Site/Site.py
index d6179307..46e19169 100644
--- a/src/Site/Site.py
+++ b/src/Site/Site.py
@@ -6,11 +6,13 @@ import time
import random
import sys
import hashlib
+import itertools
import collections
import base64
import gevent
import gevent.pool
+import gevent.lock
import util
from Config import config
@@ -27,6 +29,125 @@ from Plugin import PluginManager
from File import FileServer
from .SiteAnnouncer import SiteAnnouncer
from . import SiteManager
+from . import SiteHelpers
+
+def lerp(val_min, val_max, scale):
+ return scale * (val_max - val_min) + val_min
+
+class ScaledTimeoutHandler:
+ def __init__(self, val_min, val_max, handler=None, scaler=None):
+ self.val_min = val_min
+ self.val_max = val_max
+ self.timestamp = 0
+ self.handler = handler
+ self.scaler = scaler
+ self.log = logging.getLogger("ScaledTimeoutHandler")
+
+ def isExpired(self, scale):
+ interval = lerp(self.val_min, self.val_max, scale)
+ expired_at = self.timestamp + interval
+ now = time.time()
+ expired = (now > expired_at)
+ if expired:
+ self.log.debug(
+ "Expired: [%d..%d]: scale=%f, interval=%f",
+ self.val_min, self.val_max, scale, interval)
+ return expired
+
+ def done(self):
+ self.timestamp = time.time()
+
+ def run(self, *args, **kwargs):
+ do_run = kwargs["force"] or self.isExpired(self.scaler())
+ if do_run:
+ result = self.handler(*args, **kwargs)
+ if result:
+ self.done()
+ return result
+ else:
+ return None
+
+class BackgroundPublisher:
+ def __init__(self, site, published=[], limit=5, inner_path="content.json", diffs={}):
+ self.site = site
+ self.threads = gevent.pool.Pool(None)
+ self.inner_path = inner_path
+ self.stages = [
+ {
+ "interval": ScaledTimeoutHandler(60, 60),
+ "max_tries": 2,
+ "tries": 0,
+ "limit": 0,
+ "done": False
+ },
+ {
+ "interval": ScaledTimeoutHandler(60 * 10, 60 * 10),
+ "max_tries": 5,
+ "tries": 0,
+ "limit": 0,
+ "done": False
+ }
+ ]
+ self.reinit(published=published, limit=limit, diffs=diffs)
+
+ def reinit(self, published=[], limit=5, diffs={}):
+ self.threads.kill()
+ self.published = published
+ self.diffs = diffs
+
+ i = 0
+ for stage in self.stages:
+ stage["nr"] = i
+ stage["limit"] = limit * (2 + i)
+ stage["tries"] = 0
+ stage["done"] = False
+ stage["thread"] = None
+ if i > 0:
+ stage["interval"].done()
+ i += 1
+
+ def isStageComplete(self, stage):
+ if not stage["done"]:
+ stage["done"] = len(self.published) >= stage["limit"]
+ if not stage["done"]:
+ stage["done"] = stage["tries"] >= stage["max_tries"]
+ return stage["done"]
+
+ def isComplete(self):
+ for stage in self.stages:
+ if not self.isStageComplete(stage):
+ return False
+ return True
+
+ def process(self):
+ for stage in self.stages:
+ if not self.isStageComplete(stage):
+ self.processStage(stage)
+ break
+ return self.isComplete()
+
+ def processStage(self, stage):
+ if not stage["interval"].isExpired(0):
+ return
+
+ if len(self.site.peers) < stage["limit"]:
+ self.site.announce(mode="more")
+
+ if not stage["thread"]:
+ peers = list(self.site.peers.values())
+ random.shuffle(peers)
+ stage["thread"] = self.threads.spawn(self.site.publisher,
+ self.inner_path, peers, self.published, stage["limit"], diffs=self.diffs, max_retries=1)
+
+ stage["tries"] += 1
+ stage["interval"].done()
+
+ self.site.log.info("Background publisher: Stage #%s: %s published to %s/%s peers",
+ stage["nr"], self.inner_path, len(self.published), stage["limit"])
+
+ def finalize(self):
+ self.threads.kill()
+ self.site.log.info("Background publisher: Published %s to %s peers", self.inner_path, len(self.published))
@PluginManager.acceptPlugins
@@ -40,23 +161,35 @@ class Site(object):
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
- self.content = None # Load content.json
- self.peers = {} # Key: ip:port, Value: Peer.Peer
- self.peers_recent = collections.deque(maxlen=150)
- self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
- self.greenlet_manager = GreenletManager.GreenletManager() # Running greenlets
- self.worker_manager = WorkerManager(self) # Handle site download from other peers
- self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
- self.content_updated = None # Content.js update time
- self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
- self.page_requested = False # Page viewed in browser
- self.websockets = [] # Active site websocket connections
+ self.periodic_maintenance_handlers = [
+ ScaledTimeoutHandler(
+ config.site_announce_interval_max * 60,
+ config.site_announce_interval_min * 60,
+ handler=self.periodicMaintenanceHandler_announce,
+ scaler=self.getAnnounceRating),
+ ScaledTimeoutHandler(
+ config.site_peer_check_interval_max * 60,
+ config.site_peer_check_interval_min * 60,
+ handler=self.periodicMaintenanceHandler_peer_check,
+ scaler=self.getAnnounceRating),
+ ScaledTimeoutHandler(
+ config.site_update_check_interval_max * 60,
+ config.site_update_check_interval_min * 60,
+ handler=self.periodicMaintenanceHandler_general,
+ scaler=self.getActivityRating)
+ ]
+ self.background_publishers = {}
+ self.background_publishers_lock = gevent.lock.RLock()
+
+ # FZS = forced zero "since"
+ self.fzs_range = 20
+ self.fzs_interval = 30 * 60
+ self.fzs_count = random.randint(0, self.fzs_range / 4)
+ self.fzs_timestamp = 0
+
+ ##############################################
self.connection_server = None
- self.loadSettings(settings) # Load settings from sites.json
- self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
- self.content_manager = ContentManager(self)
- self.content_manager.loadContents() # Load content.json files
if "main" in sys.modules: # import main has side-effects, breaks tests
import main
if "file_server" in dir(main): # Use global file server by default if possible
@@ -66,9 +199,32 @@ class Site(object):
self.connection_server = main.file_server
else:
self.connection_server = FileServer()
+ ##############################################
+
+ self.content = None # Load content.json
+ self.peers = {} # Key: ip:port, Value: Peer.Peer
+ self.peers_recent = collections.deque(maxlen=150)
+ self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
+ self.greenlet_manager = GreenletManager.GreenletManager(self.connection_server.site_pool) # Running greenlets
+ self.worker_manager = WorkerManager(self) # Handle site download from other peers
+ self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
+ self.content_updated = None # Content.js update time
+ self.last_online_update = 0
+ self.startup_announce_done = 0
+ self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
+ self.page_requested = False # Page viewed in browser
+ self.websockets = [] # Active site websocket connections
+
+ self.loadSettings(settings) # Load settings from sites.json
+ self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
+ self.content_manager = ContentManager(self)
+ self.content_manager.loadContents() # Load content.json files
self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes
+ self.peer_connector = SiteHelpers.PeerConnector(self) # Connect more peers in background by request
+ self.persistent_peer_req = None # The persistent peer requirement, managed by maintenance handler
+
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
@@ -89,6 +245,10 @@ class Site(object):
settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
if settings:
self.settings = settings
+ if "check_files_timestamp" not in settings:
+ settings["check_files_timestamp"] = 0
+ if "verify_files_timestamp" not in settings:
+ settings["verify_files_timestamp"] = 0
if "cache" not in settings:
settings["cache"] = {}
if "size_files_optional" not in settings:
@@ -104,8 +264,17 @@ class Site(object):
self.bad_files[inner_path] = min(self.bad_files[inner_path], 20)
else:
self.settings = {
- "own": False, "serving": True, "permissions": [], "cache": {"bad_files": {}}, "size_files_optional": 0,
- "added": int(time.time()), "downloaded": None, "optional_downloaded": 0, "size_optional": 0
+ "check_files_timestamp": 0,
+ "verify_files_timestamp": 0,
+ "own": False,
+ "serving": True,
+ "permissions": [],
+ "cache": {"bad_files": {}},
+ "size_files_optional": 0,
+ "added": int(time.time()),
+ "downloaded": None,
+ "optional_downloaded": 0,
+ "size_optional": 0
} # Default
if config.download_optional == "auto":
self.settings["autodownloadoptional"] = True
@@ -125,12 +294,38 @@ class Site(object):
SiteManager.site_manager.load(False)
SiteManager.site_manager.saveDelayed()
+ # Returns True if any site-related activity should be interrupted
+ # due to connection server being stopped or site being deleted
+ def isStopping(self):
+ return self.connection_server.stopping or self.settings.get("deleting", False)
+
+ # Returns False if any network activity for the site should not happen
def isServing(self):
- if config.offline:
+ if self.connection_server.isOfflineMode():
+ return False
+ elif self.isStopping():
return False
else:
return self.settings["serving"]
+ # Spawns a thread that will be waited for on server being stopped (and killed after a timeout).
+ # Short cut to self.greenlet_manager.spawn()
+ def spawn(self, *args, **kwargs):
+ thread = self.greenlet_manager.spawn(*args, **kwargs)
+ return thread
+
+ # Spawns a thread that will be waited for on server being stopped (and killed after a timeout).
+ # Short cut to self.greenlet_manager.spawnLater()
+ def spawnLater(self, *args, **kwargs):
+ thread = self.greenlet_manager.spawnLater(*args, **kwargs)
+ return thread
+
+ def checkSendBackLRU(self, peer, inner_path, remote_modified):
+ return SiteManager.site_manager.checkSendBackLRU(self, peer, inner_path, remote_modified)
+
+ def addToSendBackLRU(self, peer, inner_path, modified):
+ return SiteManager.site_manager.addToSendBackLRU(self, peer, inner_path, modified)
+
def getSettingsCache(self):
back = {}
back["bad_files"] = self.bad_files
@@ -141,9 +336,32 @@ class Site(object):
def getSizeLimit(self):
return self.settings.get("size_limit", int(config.size_limit))
+ def isFileVerificationExpired(self, check_files_interval, verify_files_interval):
+ now = time.time()
+ check_files_timestamp = self.settings.get("check_files_timestamp", 0)
+ verify_files_timestamp = self.settings.get("verify_files_timestamp", 0)
+
+ if check_files_interval is None:
+ check_files_expiration = now + 1
+ else:
+ check_files_expiration = check_files_timestamp + check_files_interval
+
+ if verify_files_interval is None:
+ verify_files_expiration = now + 1
+ else:
+ verify_files_expiration = verify_files_timestamp + verify_files_interval
+
+ if verify_files_expiration < now:
+ return "verify"
+
+ if check_files_expiration < now:
+ return "check"
+
+ return False
+
# Next size limit based on current size
def getNextSizeLimit(self):
- size_limits = [25, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
+ size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
@@ -153,7 +371,7 @@ class Site(object):
def isAddedRecently(self):
return time.time() - self.settings.get("added", 0) < 60 * 60 * 24
- # Download all file from content.json
+ # Download all files from content.json
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
s = time.time()
if config.verbose:
@@ -243,7 +461,7 @@ class Site(object):
# Optionals files
if inner_path == "content.json":
- gevent.spawn(self.updateHashfield)
+ self.spawn(self.updateHashfield)
for file_relative_path in list(self.content_manager.contents[inner_path].get("files_optional", {}).keys()):
file_inner_path = content_inner_dir + file_relative_path
@@ -262,7 +480,7 @@ class Site(object):
include_threads = []
for file_relative_path in list(self.content_manager.contents[inner_path].get("includes", {}).keys()):
file_inner_path = content_inner_dir + file_relative_path
- include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
+ include_thread = self.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
if config.verbose:
@@ -282,6 +500,11 @@ class Site(object):
inner_path, time.time() - s, len(self.worker_manager.tasks)
))
+
+ # If no file tasks have been started, worker_manager.checkComplete()
+ # never called. So call it explicitly.
+ self.greenlet_manager.spawn(self.worker_manager.checkComplete)
+
return True
# Return bad files with less than 3 retry
@@ -337,9 +560,9 @@ class Site(object):
)
if self.isAddedRecently():
- gevent.spawn(self.announce, mode="start", force=True)
+ self.spawn(self.announce, mode="start", force=True)
else:
- gevent.spawn(self.announce, mode="update")
+ self.spawn(self.announce, mode="update")
if check_size: # Check the size first
valid = self.downloadContent("content.json", download_files=False) # Just download content.json files
@@ -391,14 +614,18 @@ class Site(object):
self.log.debug("Ended downloadFile pool len: %s, skipped: %s" % (len(inner_paths), num_skipped))
# Update worker, try to find client that supports listModifications command
- def updater(self, peers_try, queried, since):
+ def updater(self, peers_try, queried, need_queries, since):
threads = []
while 1:
- if not peers_try or len(queried) >= 3: # Stop after 3 successful query
+ if not peers_try or len(queried) >= need_queries: # Stop after 3 successful query
break
peer = peers_try.pop(0)
+
+ if peer in queried:
+ continue
+
if config.verbose:
- self.log.debug("CheckModifications: Try to get updates from: %s Left: %s" % (peer, peers_try))
+ self.log.debug("CheckModifications: Trying to get updates from: %s Left: %s" % (peer, peers_try))
res = None
with gevent.Timeout(20, exception=False):
@@ -408,37 +635,94 @@ class Site(object):
continue # Failed query
queried.append(peer)
+
+
modified_contents = []
- my_modified = self.content_manager.listModified(since)
- num_old_files = 0
- for inner_path, modified in res["modified_files"].items(): # Check if the peer has newer files than we
- has_newer = int(modified) > my_modified.get(inner_path, 0)
- has_older = int(modified) < my_modified.get(inner_path, 0)
- if inner_path not in self.bad_files and not self.content_manager.isArchived(inner_path, modified):
- if has_newer:
- # We dont have this file or we have older
+ send_back = []
+ send_back_limit = config.send_back_limit
+ send_back_skipped = 0
+ peer_modified_files = res["modified_files"]
+ my_modified_files = self.content_manager.listModified(since)
+
+ inner_paths = itertools.chain(peer_modified_files.keys(), my_modified_files.keys())
+ seen_inner_paths = {}
+ for inner_path in inner_paths: # Check if the peer has newer files than we have
+ if seen_inner_paths.get(inner_path, False):
+ continue
+ seen_inner_paths[inner_path] = True
+
+ peer_modified = int(peer_modified_files.get(inner_path, 0))
+ my_modified = int(my_modified_files.get(inner_path, 0))
+
+ diff = peer_modified - my_modified
+ if diff == 0:
+ continue
+ has_newer = diff > 0
+ has_older = diff < 0
+
+ if inner_path not in self.bad_files and not self.content_manager.isArchived(inner_path, peer_modified):
+ if has_newer: # We don't have this file or we have older version
modified_contents.append(inner_path)
- self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1
- if has_older and num_old_files < 5:
- num_old_files += 1
- self.log.debug("CheckModifications: %s client has older version of %s, publishing there (%s/5)..." % (peer, inner_path, num_old_files))
- gevent.spawn(self.publisher, inner_path, [peer], [], 1)
+ self.bad_files[inner_path] = self.bad_files.get(inner_path, 1)
+ if has_older: # The remote peer doesn't have this file or it has older version
+ if self.checkSendBackLRU(peer, inner_path, peer_modified):
+ send_back_skipped += 1
+ else:
+ send_back.append(inner_path)
+
+ inner_paths = None
+ seen_inner_paths = None
+
if modified_contents:
- self.log.debug("CheckModifications: %s new modified file from %s" % (len(modified_contents), peer))
- modified_contents.sort(key=lambda inner_path: 0 - res["modified_files"][inner_path]) # Download newest first
- t = gevent.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True)
+ self.log.info("CheckModifications: %s new modified files from %s" % (len(modified_contents), peer))
+ modified_contents.sort(key=lambda inner_path: 0 - peer_modified_files[inner_path]) # Download newest first
+ for inner_path in modified_contents:
+ self.log.info("CheckModifications: %s: %s > %s" % (
+ inner_path, peer_modified_files.get(inner_path, 0), my_modified_files.get(inner_path, 0)
+ ))
+ t = self.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True)
threads.append(t)
- if config.verbose:
- self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads))
+
+ if send_back:
+ self.log.info("CheckModifications: %s has older versions of %s files" % (peer, len(send_back)))
+ if len(send_back) > send_back_limit:
+ self.log.info("CheckModifications: choosing %s random files to publish back" % (send_back_limit))
+ random.shuffle(send_back)
+ send_back = send_back[0:send_back_limit]
+ for inner_path in send_back:
+ self.log.info("CheckModifications: %s: %s < %s" % (
+ inner_path, peer_modified_files.get(inner_path, 0), my_modified_files.get(inner_path, 0)
+ ))
+ self.spawn(self.publisher, inner_path, [peer], [], 1, save_to_send_back_lru=True)
+
+ if send_back_skipped:
+ self.log.info("CheckModifications: %s has older versions of %s files, skipped according to send back LRU" % (peer, send_back_skipped))
+
+ self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads))
gevent.joinall(threads)
+ # We need, with some rate, to perform the full check of modifications,
+ # "since the beginning of time", instead of the partial one.
+ def getForcedZeroSince(self):
+ now = time.time()
+ if self.fzs_timestamp + self.fzs_interval > now:
+ return False
+ self.fzs_count -= 1
+ if self.fzs_count < 1:
+ self.fzs_count = random.randint(0, self.fzs_range)
+ self.fzs_timestamp = now
+ return True
+ return False
+
# Check modified content.json files from peers and add modified files to bad_files
# Return: Successfully queried peers [Peer, Peer...]
def checkModifications(self, since=None):
s = time.time()
peers_try = [] # Try these peers
queried = [] # Successfully queried from these peers
- limit = 5
+ peer_limit = 10
+ updater_limit = 3
+ need_queries = 3
# Wait for peers
if not self.peers:
@@ -449,58 +733,116 @@ class Site(object):
if self.peers:
break
- peers_try = self.getConnectedPeers()
- peers_connected_num = len(peers_try)
- if peers_connected_num < limit * 2: # Add more, non-connected peers if necessary
- peers_try += self.getRecentPeers(limit * 5)
+ if since is None:
+ if self.getForcedZeroSince():
+ since = 0
+ else:
+ margin = 60 * 60 * 24
+ since = self.settings.get("modified", margin) - margin
- if since is None: # No since defined, download from last modification time-1day
- since = self.settings.get("modified", 60 * 60 * 24) - 60 * 60 * 24
+ if since == 0:
+ peer_limit *= 4
+ need_queries *= 4
- if config.verbose:
- self.log.debug(
- "CheckModifications: Try to get listModifications from peers: %s, connected: %s, since: %s" %
- (peers_try, peers_connected_num, since)
- )
+ peers_try = self.getConnectedPeers() + self.getConnectablePeers(peer_limit)
+
+ self.log.debug(
+ "CheckModifications: Trying to get listModifications from %s peers, %s connected, since: %s" %
+ (len(peers_try), len(self.getConnectedPeers()), since)
+ )
updaters = []
- for i in range(3):
- updaters.append(gevent.spawn(self.updater, peers_try, queried, since))
+ for i in range(updater_limit):
+ updaters.append(self.spawn(self.updater, peers_try, queried, need_queries, since))
- gevent.joinall(updaters, timeout=10) # Wait 10 sec to workers done query modifications
+ for r in range(10):
+ gevent.joinall(updaters, timeout=5+r)
+ if len(queried) >= need_queries or len(peers_try) == 0:
+ break
+ self.log.debug("CheckModifications: Waiting... (%s) succesfully queried: %s, left: %s" %
+ (r + 1, len(queried), len(peers_try)))
- if not queried: # Start another 3 thread if first 3 is stuck
- peers_try[0:0] = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Add connected peers
- for _ in range(10):
- gevent.joinall(updaters, timeout=10) # Wait another 10 sec if none of updaters finished
- if queried:
- break
-
- self.log.debug("CheckModifications: Queried listModifications from: %s in %.3fs since %s" % (queried, time.time() - s, since))
+ self.log.debug("CheckModifications: Queried listModifications from %s peers in %.3fs since %s" % (
+ len(queried), time.time() - s, since))
time.sleep(0.1)
return queried
+ def invalidateUpdateTime(self, invalid_interval):
+ a, b = invalid_interval
+ if b is None:
+ b = time.time()
+ if a is None:
+ a = b
+ if a <= self.last_online_update and self.last_online_update <= b:
+ self.last_online_update = 0
+ self.log.debug("Update time invalidated")
+
+ def isUpdateTimeValid(self):
+ if not self.last_online_update:
+ return False
+ expirationThreshold = 60 * 60 * 6
+ return self.last_online_update > time.time() - expirationThreshold
+
+ def refreshUpdateTime(self, valid=True):
+ if valid:
+ self.last_online_update = time.time()
+ else:
+ self.last_online_update = 0
+
# Update content.json from peers and download changed files
# Return: None
@util.Noparallel()
- def update(self, announce=False, check_files=False, since=None):
+ def update(self, announce=False, check_files=False, verify_files=False, since=None):
+ online = self.connection_server.isInternetOnline()
+
self.content_manager.loadContent("content.json", load_includes=False) # Reload content.json
self.content_updated = None # Reset content updated time
+ if verify_files:
+ check_files = True
+
+ if verify_files:
+ self.updateWebsocket(verifying=True)
+ elif check_files:
+ self.updateWebsocket(checking=True)
if check_files:
- self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size
+ if verify_files:
+ self.storage.updateBadFiles(quick_check=False) # Full-featured checksum verification
+ else:
+ self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size
+ # Don't update the timestamps in case of the application being shut down,
+ # so we can make another try next time.
+ if not self.isStopping():
+ self.settings["check_files_timestamp"] = time.time()
+ if verify_files:
+ self.settings["verify_files_timestamp"] = time.time()
+
+ if verify_files:
+ self.updateWebsocket(verified=True)
+ elif check_files:
+ self.updateWebsocket(checked=True)
if not self.isServing():
return False
+ if announce:
+ self.updateWebsocket(updating=True)
+ self.announce(mode="update", force=True)
+
+ reqs = [
+ self.peer_connector.newReq(4, 4, 30),
+ self.peer_connector.newReq(2, 2, 60),
+ self.peer_connector.newReq(1, 1, 120)
+ ]
+ nr_connected_peers = self.waitForPeers(reqs);
+ if nr_connected_peers < 1:
+ return
+
self.updateWebsocket(updating=True)
# Remove files that no longer in content.json
self.checkBadFiles()
- if announce:
- self.announce(mode="update", force=True)
-
# Full update, we can reset bad files
if check_files and since == 0:
self.bad_files = {}
@@ -511,7 +853,7 @@ class Site(object):
if self.bad_files:
self.log.debug("Bad files: %s" % self.bad_files)
- gevent.spawn(self.retryBadFiles, force=True)
+ self.spawn(self.retryBadFiles, force=True)
if len(queried) == 0:
# Failed to query modifications
@@ -519,8 +861,19 @@ class Site(object):
else:
self.content_updated = time.time()
+ self.sendMyHashfield()
+ self.updateHashfield()
+
+ online = online and self.connection_server.isInternetOnline()
+ self.refreshUpdateTime(valid=online)
+
self.updateWebsocket(updated=True)
+ # To be called from FileServer
+ @util.Noparallel(queue=True, ignore_args=True)
+ def update2(self, check_files=False, verify_files=False):
+ self.update(check_files=check_files, verify_files=verify_files)
+
# Update site by redownload all content.json
def redownloadContents(self):
# Download all content.json again
@@ -532,7 +885,7 @@ class Site(object):
gevent.joinall(content_threads)
# Publish worker
- def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None):
+ def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None, max_retries=2, save_to_send_back_lru=False):
file_size = self.storage.getSize(inner_path)
content_json_modified = self.content_manager.contents[inner_path]["modified"]
body = self.storage.read(inner_path)
@@ -557,7 +910,7 @@ class Site(object):
timeout = 10 + int(file_size / 1024)
result = {"exception": "Timeout"}
- for retry in range(2):
+ for retry in range(max_retries):
try:
with gevent.Timeout(timeout, False):
result = peer.publish(self.address, inner_path, body, content_json_modified, diffs)
@@ -567,6 +920,12 @@ class Site(object):
self.log.error("Publish error: %s" % Debug.formatException(err))
result = {"exception": Debug.formatException(err)}
+ # We add to the send back lru not only on success, but also on errors.
+ # Some peers returns None. (Why?)
+ # Anyway, we tried our best in delivering possibly lost updates.
+ if save_to_send_back_lru:
+ self.addToSendBackLRU(peer, inner_path, content_json_modified)
+
if result and "ok" in result:
published.append(peer)
if cb_progress and len(published) <= limit:
@@ -578,40 +937,75 @@ class Site(object):
self.log.info("[FAILED] %s: %s" % (peer.key, result))
time.sleep(0.01)
+ def addBackgroundPublisher(self, published=[], limit=5, inner_path="content.json", diffs={}):
+ with self.background_publishers_lock:
+ if self.background_publishers.get(inner_path, None):
+ background_publisher = self.background_publishers[inner_path]
+ background_publisher.reinit(published=published, limit=limit, diffs=diffs)
+ else:
+ background_publisher = BackgroundPublisher(self, published=published, limit=limit, inner_path=inner_path, diffs=diffs)
+ self.background_publishers[inner_path] = background_publisher
+
+ self.spawn(background_publisher.process)
+
+ def processBackgroundPublishers(self):
+ with self.background_publishers_lock:
+ for inner_path, background_publisher in list(self.background_publishers.items()):
+ background_publisher.process()
+ if background_publisher.isComplete():
+ background_publisher.finalize()
+ del self.background_publishers[inner_path]
+
+ def getPeersForForegroundPublishing(self, limit):
+ # Wait for some peers to appear
+ reqs = [
+ self.peer_connector.newReq(limit, limit / 2, 10), # some of them...
+ self.peer_connector.newReq(1, 1, 60) # or at least one...
+ ]
+ self.waitForPeers(reqs, update_websocket=False)
+
+ peers = self.getConnectedPeers()
+ random.shuffle(peers)
+
+ # Prefer newer clients.
+ # Trying to deliver foreground updates to the latest version clients,
+ # expecting that they have better networking facilities.
+ # Note: background updates SHOULD NOT discriminate peers by their rev number,
+ # otherwise it can cause troubles in delivering updates to older versions.
+ peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100)
+
+ # Add more, non-connected peers if necessary
+ if len(peers) < limit * 2 and len(self.peers) > len(peers):
+ peers += self.getRecentPeers(limit * 2)
+ peers = set(peers)
+
+ return peers
+
# Update content.json on peers
@util.Noparallel()
def publish(self, limit="default", inner_path="content.json", diffs={}, cb_progress=None):
published = [] # Successfully published (Peer)
publishers = [] # Publisher threads
- if not self.peers:
- self.announce(mode="more")
-
if limit == "default":
limit = 5
threads = limit
- peers = self.getConnectedPeers()
- num_connected_peers = len(peers)
-
- random.shuffle(peers)
- peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) # Prefer newer clients
-
- if len(peers) < limit * 2 and len(self.peers) > len(peers): # Add more, non-connected peers if necessary
- peers += self.getRecentPeers(limit * 2)
-
- peers = set(peers)
+ peers = self.getPeersForForegroundPublishing(limit)
self.log.info("Publishing %s to %s/%s peers (connected: %s) diffs: %s (%.2fk)..." % (
- inner_path, limit, len(self.peers), num_connected_peers, list(diffs.keys()), float(len(str(diffs))) / 1024
+ inner_path,
+ limit, len(self.peers), len(self.getConnectedPeers()),
+ list(diffs.keys()), float(len(str(diffs))) / 1024
))
if not peers:
+ self.addBackgroundPublisher(published=published, limit=limit, inner_path=inner_path, diffs=diffs)
return 0 # No peers found
event_done = gevent.event.AsyncResult()
for i in range(min(len(peers), limit, threads)):
- publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, diffs, event_done, cb_progress)
+ publisher = self.spawn(self.publisher, inner_path, peers, published, limit, diffs, event_done, cb_progress)
publishers.append(publisher)
event_done.get() # Wait for done
@@ -620,17 +1014,16 @@ class Site(object):
if len(published) == 0:
gevent.joinall(publishers) # No successful publish, wait for all publisher
- # Publish more peers in the backgroup
+ # Publish to more peers in the background
self.log.info(
- "Published %s to %s peers, publishing to %s more peers in the background" %
- (inner_path, len(published), limit)
+ "Published %s to %s peers, publishing to more peers in the background" %
+ (inner_path, len(published))
)
- for thread in range(2):
- gevent.spawn(self.publisher, inner_path, peers, published, limit=limit * 2, diffs=diffs)
+ self.addBackgroundPublisher(published=published, limit=limit, inner_path=inner_path, diffs=diffs)
# Send my hashfield to every connected peer if changed
- gevent.spawn(self.sendMyHashfield, 100)
+ self.spawn(self.sendMyHashfield, 100)
return len(published)
@@ -793,7 +1186,7 @@ class Site(object):
if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
self.log.debug("Need content.json first (inner_path: %s, priority: %s)" % (inner_path, priority))
if priority > 0:
- gevent.spawn(self.announce)
+ self.spawn(self.announce)
if inner_path != "content.json": # Prevent double download
task = self.worker_manager.addTask("content.json", peer)
task["evt"].get()
@@ -819,7 +1212,7 @@ class Site(object):
self.log.debug("%s: Download not allowed" % inner_path)
return False
- self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file
+ self.bad_files[inner_path] = self.bad_files.get(inner_path, 1) # Mark as bad file
task = self.worker_manager.addTask(inner_path, peer, priority=priority, file_info=file_info)
if blocking:
@@ -847,118 +1240,239 @@ class Site(object):
peer = Peer(ip, port, self)
self.peers[key] = peer
peer.found(source)
+
+ self.peer_connector.processReqs()
+ self.peer_connector.addPeer(peer)
+
return peer
+ # Called from peer.remove to erase links to peer
+ def deregisterPeer(self, peer):
+ self.peers.pop(peer.key, None)
+ try:
+ self.peers_recent.remove(peer)
+ except:
+ pass
+ self.peer_connector.deregisterPeer(peer)
+
def announce(self, *args, **kwargs):
if self.isServing():
self.announcer.announce(*args, **kwargs)
- # Keep connections to get the updates
- def needConnections(self, num=None, check_site_on_reconnect=False):
+ def getActivityRating(self, force_safe=False):
+ age = time.time() - self.settings.get("modified", 0)
+
+ if age < 60 * 60:
+ rating = 1.0
+ elif age < 60 * 60 * 5:
+ rating = 0.8
+ elif age < 60 * 60 * 24:
+ rating = 0.6
+ elif age < 60 * 60 * 24 * 3:
+ rating = 0.4
+ elif age < 60 * 60 * 24 * 7:
+ rating = 0.2
+ else:
+ rating = 0.0
+
+ force_safe = force_safe or config.expose_no_ownership
+
+ if (not force_safe) and self.settings["own"]:
+ rating = min(rating, 0.6)
+
+ if self.content_updated is False: # Last check modification failed
+ rating += 0.1
+ elif self.bad_files:
+ rating += 0.1
+
+ if rating > 1.0:
+ rating = 1.0
+
+ return rating
+
+ def getAnnounceRating(self):
+ # rare frequent
+ # announces announces
+ # 0 ------------------- 1
+ # activity -------------> -- active site ==> frequent announces
+ # <---------------- peers -- many peers ==> rare announces
+ # trackers -------------> -- many trackers ==> frequent announces to iterate over more trackers
+
+ activity_rating = self.getActivityRating(force_safe=True)
+
+ peer_count = len(self.peers)
+ peer_rating = 1.0 - min(peer_count, 50) / 50.0
+
+ tracker_count = self.announcer.getSupportedTrackerCount()
+ tracker_count = max(tracker_count, 1)
+ tracker_rating = 1.0 - (1.0 / tracker_count)
+
+ v = [activity_rating, peer_rating, tracker_rating]
+ return sum(v) / float(len(v))
+
+ def getPreferableConnectablePeerCount(self):
+ if not self.isServing():
+ return 0
+
+ count = lerp(
+ config.site_connectable_peer_count_min,
+ config.site_connectable_peer_count_max,
+ self.getActivityRating(force_safe=True))
+ return count
+
+ # The engine tries to maintain the number of active connections:
+ # >= getPreferableActiveConnectionCount()
+ # and
+ # <= getActiveConnectionCountLimit()
+
+ def getPreferableActiveConnectionCount(self):
+ if not self.isServing():
+ return 0
+
+ age = time.time() - self.settings.get("modified", 0)
+ count = int(10 * self.getActivityRating(force_safe=True))
+
+ if len(self.peers) < 50:
+ count = max(count, 5)
+
+ count = min(count, config.connected_limit)
+
+ return count
+
+ def getActiveConnectionCountLimit(self):
+ count_above_preferable = 2
+ limit = self.getPreferableActiveConnectionCount() + count_above_preferable
+ limit = min(limit, config.connected_limit)
+ return limit
+
+ ############################################################################
+
+ # Returns the maximum value of current reqs for connections
+ def waitingForConnections(self):
+ self.peer_connector.processReqs()
+ return self.peer_connector.need_nr_connected_peers
+
+ def needConnections(self, num=None, update_site_on_reconnect=False):
+ if not self.connection_server.allowsCreatingConnections():
+ return
+
if num is None:
- if len(self.peers) < 50:
- num = 3
- else:
- num = 6
- need = min(len(self.peers), num, config.connected_limit) # Need 5 peer, but max total peers
+ num = self.getPreferableActiveConnectionCount()
+ num = min(len(self.peers), num)
- connected = len(self.getConnectedPeers())
+ req = self.peer_connector.newReq(0, num)
+ return req
- connected_before = connected
+ # Wait for peers to be discovered and/or connected according to reqs
+ # and send updates to the UI
+ def waitForPeers(self, reqs, update_websocket=True):
+ if not reqs:
+ return 0
+ i = 0
+ nr_connected_peers = -1
+ while self.isServing():
+ ready_reqs = list(filter(lambda req: req.ready(), reqs))
+ if len(ready_reqs) == len(reqs):
+ if nr_connected_peers < 0:
+ nr_connected_peers = ready_reqs[0].nr_connected_peers
+ break
+ waiting_reqs = list(filter(lambda req: not req.ready(), reqs))
+ if not waiting_reqs:
+ break
+ waiting_req = waiting_reqs[0]
+ #self.log.debug("waiting_req: %s %s %s", waiting_req.need_nr_connected_peers, waiting_req.nr_connected_peers, waiting_req.expiration_interval)
+ waiting_req.waitHeartbeat(timeout=1.0)
+ if i > 0 and nr_connected_peers != waiting_req.nr_connected_peers:
+ nr_connected_peers = waiting_req.nr_connected_peers
+ if update_websocket:
+ self.updateWebsocket(connecting_to_peers=nr_connected_peers)
+ i += 1
+ if update_websocket:
+ self.updateWebsocket(connected_to_peers=max(nr_connected_peers, 0))
+ if i > 1:
+ # If we waited some time, pause now for displaying connected_to_peers message in the UI.
+ # This sleep is solely needed for site status updates on ZeroHello to be more cool-looking.
+ gevent.sleep(1)
+ return nr_connected_peers
- self.log.debug("Need connections: %s, Current: %s, Total: %s" % (need, connected, len(self.peers)))
+ ############################################################################
- if connected < need: # Need more than we have
- for peer in self.getRecentPeers(30):
- if not peer.connection or not peer.connection.connected: # No peer connection or disconnected
- peer.pex() # Initiate peer exchange
- if peer.connection and peer.connection.connected:
- connected += 1 # Successfully connected
- if connected >= need:
- break
- self.log.debug(
- "Connected before: %s, after: %s. Check site: %s." %
- (connected_before, connected, check_site_on_reconnect)
- )
-
- if check_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet:
- gevent.spawn(self.update, check_files=False)
-
- return connected
-
- # Return: Probably peers verified to be connectable recently
+ # Return: Peers verified to be connectable recently, or if not enough, other peers as well
def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True):
peers = list(self.peers.values())
- found = []
+ random.shuffle(peers)
+ connectable_peers = []
+ reachable_peers = []
for peer in peers:
- if peer.key.endswith(":0"):
- continue # Not connectable
- if not peer.connection:
- continue # No connection
- if peer.ip.endswith(".onion") and not self.connection_server.tor_manager.enabled:
- continue # Onion not supported
if peer.key in ignore:
- continue # The requester has this peer
- if time.time() - peer.connection.last_recv_time > 60 * 60 * 2: # Last message more than 2 hours ago
- peer.connection = None # Cleanup: Dead connection
continue
if not allow_private and helper.isPrivateIp(peer.ip):
continue
- found.append(peer)
- if len(found) >= need_num:
+ if peer.isConnectable():
+ connectable_peers.append(peer)
+ elif peer.isReachable():
+ reachable_peers.append(peer)
+ if len(connectable_peers) >= need_num:
break # Found requested number of peers
- if len(found) < need_num: # Return not that good peers
- found += [
- peer for peer in peers
- if not peer.key.endswith(":0") and
- peer.key not in ignore and
- (allow_private or not helper.isPrivateIp(peer.ip))
- ][0:need_num - len(found)]
+ if len(connectable_peers) < need_num: # Return not that good peers
+ connectable_peers += reachable_peers[0:need_num - len(connectable_peers)]
- return found
+ return connectable_peers
# Return: Recently found peers
+ def getReachablePeers(self):
+ return [peer for peer in self.peers.values() if peer.isReachable()]
+
+ # Return: Recently found peers, sorted by reputation.
+ # If there not enough recently found peers, adds other known peers with highest reputation
def getRecentPeers(self, need_num):
- found = list(set(self.peers_recent))
+ need_num = int(need_num)
+ found = set(self.peers_recent)
self.log.debug(
"Recent peers %s of %s (need: %s)" %
(len(found), len(self.peers), need_num)
)
- if len(found) >= need_num or len(found) >= len(self.peers):
- return sorted(
- found,
+ if len(found) < need_num and len(found) < len(self.peers):
+ # Add random peers
+ peers = self.getReachablePeers()
+ peers = sorted(
+ list(peers),
key=lambda peer: peer.reputation,
reverse=True
- )[0:need_num]
+ )
+ while len(found) < need_num and len(peers) > 0:
+ found.add(peers.pop())
- # Add random peers
- need_more = need_num - len(found)
- if not self.connection_server.tor_manager.enabled:
- peers = [peer for peer in self.peers.values() if not peer.ip.endswith(".onion")]
- else:
- peers = list(self.peers.values())
-
- found_more = sorted(
- peers[0:need_more * 50],
+ return sorted(
+ list(found),
key=lambda peer: peer.reputation,
reverse=True
- )[0:need_more * 2]
-
- found += found_more
+ )[0:need_num]
return found[0:need_num]
- def getConnectedPeers(self):
+ # Returns the list of connected peers
+ # By default the result may contain peers chosen optimistically:
+ # If the connection is being established and 20 seconds have not yet passed
+ # since the connection start time, those peers are included in the result.
+ # Set only_fully_connected=True for restricting only by fully connected peers.
+ def getConnectedPeers(self, only_fully_connected=False):
back = []
if not self.connection_server:
return []
tor_manager = self.connection_server.tor_manager
for connection in self.connection_server.connections:
+ if len(back) >= len(self.peers): # short cut for breaking early; no peers to check left
+ break
+
if not connection.connected and time.time() - connection.start_time > 20: # Still not connected after 20s
continue
+ if not connection.connected and only_fully_connected: # Only fully connected peers
+ continue
+
peer = self.peers.get("%s:%s" % (connection.ip, connection.port))
if peer:
if connection.ip.endswith(".onion") and connection.target_onion and tor_manager.start_onions:
@@ -971,61 +1485,158 @@ class Site(object):
back.append(peer)
return back
- # Cleanup probably dead peers and close connection if too much
- def cleanupPeers(self, peers_protected=[]):
+ def removeDeadPeers(self):
peers = list(self.peers.values())
- if len(peers) > 20:
- # Cleanup old peers
- removed = 0
- if len(peers) > 1000:
- ttl = 60 * 60 * 1
- else:
- ttl = 60 * 60 * 4
+ if len(peers) <= 20:
+ return
- for peer in peers:
- if peer.connection and peer.connection.connected:
- continue
- if peer.connection and not peer.connection.connected:
- peer.connection = None # Dead connection
- if time.time() - peer.time_found > ttl: # Not found on tracker or via pex in last 4 hour
- peer.remove("Time found expired")
- removed += 1
- if removed > len(peers) * 0.1: # Don't remove too much at once
- break
+ removed = 0
+ if len(peers) > 10000:
+ ttl = 60 * 2
+ elif len(peers) > 1000:
+ ttl = 60 * 60 * 1
+ elif len(peers) > 100:
+ ttl = 60 * 60 * 4
+ else:
+ ttl = 60 * 60 * 8
- if removed:
- self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers)))
+ for peer in peers:
+ if peer.isConnected() or peer.isProtected():
+ continue
+ if peer.isTtlExpired(ttl):
+ peer.remove("TTL expired")
+ removed += 1
+ if removed > len(peers) * 0.1: # Don't remove too much at once
+ break
- # Close peers over the limit
- closed = 0
- connected_peers = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Only fully connected peers
- need_to_close = len(connected_peers) - config.connected_limit
+ if removed:
+ self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers)))
- if closed < need_to_close:
- # Try to keep connections with more sites
+ # Cleanup probably dead peers and close connection if too much
+ def cleanupPeers(self):
+ self.removeDeadPeers()
+
+ limit = max(self.getActiveConnectionCountLimit(), self.waitingForConnections())
+ connected_peers = self.getConnectedPeers(only_fully_connected=True)
+ need_to_close = len(connected_peers) - limit
+
+ if need_to_close > 0:
+ closed = 0
for peer in sorted(connected_peers, key=lambda peer: min(peer.connection.sites, 5)):
- if not peer.connection:
+ if not peer.isConnected():
continue
- if peer.key in peers_protected:
+ if peer.isProtected():
continue
if peer.connection.sites > 5:
break
- peer.connection.close("Cleanup peers")
- peer.connection = None
+ peer.disconnect("Cleanup peers")
closed += 1
if closed >= need_to_close:
break
- if need_to_close > 0:
- self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (len(connected_peers), need_to_close, closed))
+ self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (
+ len(connected_peers), need_to_close, closed))
+
+ def lookForConnectablePeers(self):
+ num_tries = 2
+ need_connectable_peers = self.getPreferableConnectablePeerCount()
+
+ connectable_peers = 0
+ reachable_peers = []
+
+ for peer in list(self.peers.values()):
+ if peer.isConnected() or peer.isConnectable():
+ connectable_peers += 1
+ elif peer.isReachable():
+ reachable_peers.append(peer)
+ if connectable_peers >= need_connectable_peers:
+ return True
+
+ random.shuffle(reachable_peers)
+
+ for peer in reachable_peers:
+ if peer.isConnected() or peer.isConnectable() or peer.removed:
+ continue
+ peer.ping()
+ if peer.isConnected():
+ peer.pex()
+ num_tries -= 1
+ if num_tries < 1:
+ break
+
+ @util.Noparallel(queue=True)
+ def runPeriodicMaintenance(self, startup=False, force=False):
+ if not self.isServing():
+ return False
+
+ self.log.debug("runPeriodicMaintenance: startup=%s, force=%s" % (startup, force))
+
+ result = False
+
+ for handler in self.periodic_maintenance_handlers:
+ result = result | bool(handler.run(startup=startup, force=force))
+
+ return result
+
+ def periodicMaintenanceHandler_general(self, startup=False, force=False):
+ if not self.isServing():
+ return False
+
+ if not self.peers:
+ return False
+
+ self.log.debug("periodicMaintenanceHandler_general: startup=%s, force=%s" % (startup, force))
+
+ #self.persistent_peer_req = self.needConnections(update_site_on_reconnect=True)
+ #self.persistent_peer_req.result_connected.wait(timeout=2.0)
+
+ #self.announcer.announcePex()
+
+ self.processBackgroundPublishers()
+
+ self.update()
+
+ return True
+
+ def periodicMaintenanceHandler_peer_check(self, startup=False, force=False):
+ if not self.isServing():
+ return False
+
+ if not self.peers:
+ return False
+
+ self.log.debug("periodicMaintenanceHandler_peer_check: startup=%s, force=%s" % (startup, force))
+
+ if not startup:
+ self.cleanupPeers()
+
+ self.lookForConnectablePeers()
+
+ return True
+
+ def periodicMaintenanceHandler_announce(self, startup=False, force=False):
+ if not self.isServing():
+ return False
+
+ self.log.debug("periodicMaintenanceHandler_announce: startup=%s, force=%s" % (startup, force))
+
+ if startup and len(self.peers) < 10:
+ self.announce(mode="startup")
+ else:
+ self.announce(mode="update", pex=False)
+
+ return True
# Send hashfield to peers
def sendMyHashfield(self, limit=5):
+ if not self.isServing():
+ return False
+
if not self.content_manager.hashfield: # No optional files
return False
sent = 0
- connected_peers = self.getConnectedPeers()
+ connected_peers = self.getConnectedPeers(only_fully_connected=True)
for peer in connected_peers:
if peer.sendMyHashfield():
sent += 1
@@ -1038,13 +1649,16 @@ class Site(object):
# Update hashfield
def updateHashfield(self, limit=5):
+ if not self.isServing():
+ return False
+
# Return if no optional files
if not self.content_manager.hashfield and not self.content_manager.has_optional_files:
return False
s = time.time()
queried = 0
- connected_peers = self.getConnectedPeers()
+ connected_peers = self.getConnectedPeers(only_fully_connected=True)
for peer in connected_peers:
if peer.time_hashfield:
continue
@@ -1082,10 +1696,10 @@ class Site(object):
# Add event listeners
def addEventListeners(self):
- self.onFileStart = util.Event() # If WorkerManager added new task
+ self.onFileStart = util.Event() # If WorkerManager added new task
self.onFileDone = util.Event() # If WorkerManager successfully downloaded a file
self.onFileFail = util.Event() # If WorkerManager failed to download a file
- self.onComplete = util.Event() # All file finished
+ self.onComplete = util.Event() # All files finished
self.onFileStart.append(lambda inner_path: self.fileStarted()) # No parameters to make Noparallel batching working
self.onFileDone.append(lambda inner_path: self.fileDone(inner_path))
@@ -1099,6 +1713,7 @@ class Site(object):
param = None
for ws in self.websockets:
ws.event("siteChanged", self, param)
+ time.sleep(0.001)
def messageWebsocket(self, message, type="info", progress=None):
for ws in self.websockets:
@@ -1113,7 +1728,7 @@ class Site(object):
time.sleep(0.001) # Wait for other files adds
self.updateWebsocket(file_started=True)
- # File downloaded successful
+ # File downloaded successfully
def fileDone(self, inner_path):
# File downloaded, remove it from bad files
if inner_path in self.bad_files:
diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py
index 2fd63e82..1baf39af 100644
--- a/src/Site/SiteAnnouncer.py
+++ b/src/Site/SiteAnnouncer.py
@@ -1,6 +1,7 @@
import random
import time
import hashlib
+import logging
import re
import collections
@@ -12,6 +13,7 @@ from Debug import Debug
from util import helper
from greenlet import GreenletExit
import util
+from util import CircularIterator
class AnnounceError(Exception):
@@ -24,11 +26,20 @@ global_stats = collections.defaultdict(lambda: collections.defaultdict(int))
class SiteAnnouncer(object):
def __init__(self, site):
self.site = site
+ self.log = logging.getLogger("Site:%s SiteAnnouncer" % self.site.address_short)
+
self.stats = {}
self.fileserver_port = config.fileserver_port
self.peer_id = self.site.connection_server.peer_id
- self.last_tracker_id = random.randint(0, 10)
+ self.tracker_circular_iterator = CircularIterator()
self.time_last_announce = 0
+ self.supported_tracker_count = 0
+
+ # Returns connection_server rela
+ # Since 0.8.0
+ @property
+ def connection_server(self):
+ return self.site.connection_server
def getTrackers(self):
return config.trackers
@@ -36,25 +47,76 @@ class SiteAnnouncer(object):
def getSupportedTrackers(self):
trackers = self.getTrackers()
- if not self.site.connection_server.tor_manager.enabled:
+ if not self.connection_server.tor_manager.enabled:
trackers = [tracker for tracker in trackers if ".onion" not in tracker]
trackers = [tracker for tracker in trackers if self.getAddressParts(tracker)] # Remove trackers with unknown address
- if "ipv6" not in self.site.connection_server.supported_ip_types:
- trackers = [tracker for tracker in trackers if helper.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"]
+ if "ipv6" not in self.connection_server.supported_ip_types:
+ trackers = [tracker for tracker in trackers if self.connection_server.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"]
return trackers
- def getAnnouncingTrackers(self, mode):
+ # Returns a cached value of len(self.getSupportedTrackers()), which can be
+ # inacurate.
+ # To be used from Site for estimating available tracker count.
+ def getSupportedTrackerCount(self):
+ return self.supported_tracker_count
+
+ def shouldTrackerBeTemporarilyIgnored(self, tracker, mode, force):
+ if not tracker:
+ return True
+
+ if force:
+ return False
+
+ now = time.time()
+
+ # Throttle accessing unresponsive trackers
+ tracker_stats = global_stats[tracker]
+ delay = min(30 * tracker_stats["num_error"], 60 * 10)
+ time_announce_allowed = tracker_stats["time_request"] + delay
+ if now < time_announce_allowed:
+ return True
+
+ return False
+
+ def getAnnouncingTrackers(self, mode, force):
trackers = self.getSupportedTrackers()
- if trackers and (mode == "update" or mode == "more"): # Only announce on one tracker, increment the queried tracker id
- self.last_tracker_id += 1
- self.last_tracker_id = self.last_tracker_id % len(trackers)
- trackers_announcing = [trackers[self.last_tracker_id]] # We only going to use this one
+ self.supported_tracker_count = len(trackers)
+
+ if trackers and (mode == "update" or mode == "more"):
+
+ # Choose just 2 trackers to announce to
+
+ trackers_announcing = []
+
+ # One is the next in sequence
+
+ self.tracker_circular_iterator.resetSuccessiveCount()
+ while 1:
+ tracker = self.tracker_circular_iterator.next(trackers)
+ if not self.shouldTrackerBeTemporarilyIgnored(tracker, mode, force):
+ trackers_announcing.append(tracker)
+ break
+ if self.tracker_circular_iterator.isWrapped():
+ break
+
+ # And one is just random
+
+ shuffled_trackers = random.sample(trackers, len(trackers))
+ for tracker in shuffled_trackers:
+ if tracker in trackers_announcing:
+ continue
+ if not self.shouldTrackerBeTemporarilyIgnored(tracker, mode, force):
+ trackers_announcing.append(tracker)
+ break
else:
- trackers_announcing = trackers
+ trackers_announcing = [
+ tracker for tracker in trackers
+ if not self.shouldTrackerBeTemporarilyIgnored(tracker, mode, force)
+ ]
return trackers_announcing
@@ -62,94 +124,32 @@ class SiteAnnouncer(object):
back = []
# Type of addresses they can reach me
if config.trackers_proxy == "disable" and config.tor != "always":
- for ip_type, opened in list(self.site.connection_server.port_opened.items()):
+ for ip_type, opened in list(self.connection_server.port_opened.items()):
if opened:
back.append(ip_type)
- if self.site.connection_server.tor_manager.start_onions:
+ if self.connection_server.tor_manager.start_onions:
back.append("onion")
return back
- @util.Noparallel(blocking=False)
+ @util.Noparallel()
def announce(self, force=False, mode="start", pex=True):
+ if not self.site.isServing():
+ return
+
if time.time() - self.time_last_announce < 30 and not force:
return # No reannouncing within 30 secs
- if force:
- self.site.log.debug("Force reannounce in mode %s" % mode)
+
+ self.log.debug("announce: force=%s, mode=%s, pex=%s" % (force, mode, pex))
self.fileserver_port = config.fileserver_port
self.time_last_announce = time.time()
- trackers = self.getAnnouncingTrackers(mode)
-
- if config.verbose:
- self.site.log.debug("Tracker announcing, trackers: %s" % trackers)
-
- errors = []
- slow = []
- s = time.time()
- threads = []
- num_announced = 0
-
- for tracker in trackers: # Start announce threads
- tracker_stats = global_stats[tracker]
- # Reduce the announce time for trackers that looks unreliable
- time_announce_allowed = time.time() - 60 * min(30, tracker_stats["num_error"])
- if tracker_stats["num_error"] > 5 and tracker_stats["time_request"] > time_announce_allowed and not force:
- if config.verbose:
- self.site.log.debug("Tracker %s looks unreliable, announce skipped (error: %s)" % (tracker, tracker_stats["num_error"]))
- continue
- thread = self.site.greenlet_manager.spawn(self.announceTracker, tracker, mode=mode)
- threads.append(thread)
- thread.tracker = tracker
-
- time.sleep(0.01)
- self.updateWebsocket(trackers="announcing")
-
- gevent.joinall(threads, timeout=20) # Wait for announce finish
-
- for thread in threads:
- if thread.value is None:
- continue
- if thread.value is not False:
- if thread.value > 1.0: # Takes more than 1 second to announce
- slow.append("%.2fs %s" % (thread.value, thread.tracker))
- num_announced += 1
- else:
- if thread.ready():
- errors.append(thread.tracker)
- else: # Still running
- slow.append("30s+ %s" % thread.tracker)
-
- # Save peers num
- self.site.settings["peers"] = len(self.site.peers)
-
- if len(errors) < len(threads): # At least one tracker finished
- if len(trackers) == 1:
- announced_to = trackers[0]
- else:
- announced_to = "%s/%s trackers" % (num_announced, len(threads))
- if mode != "update" or config.verbose:
- self.site.log.debug(
- "Announced in mode %s to %s in %.3fs, errors: %s, slow: %s" %
- (mode, announced_to, time.time() - s, errors, slow)
- )
- else:
- if len(threads) > 1:
- self.site.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s))
- if len(threads) == 1 and mode != "start": # Move to next tracker
- self.site.log.debug("Tracker failed, skipping to next one...")
- self.site.greenlet_manager.spawnLater(1.0, self.announce, force=force, mode=mode, pex=pex)
-
- self.updateWebsocket(trackers="announced")
+ trackers = self.getAnnouncingTrackers(mode, force)
+ self.log.debug("Chosen trackers: %s" % trackers)
+ self.announceToTrackers(trackers, force=force, mode=mode)
if pex:
- self.updateWebsocket(pex="announcing")
- if mode == "more": # Need more peers
- self.announcePex(need_num=10)
- else:
- self.announcePex()
-
- self.updateWebsocket(pex="announced")
+ self.announcePex()
def getTrackerHandler(self, protocol):
return None
@@ -177,7 +177,7 @@ class SiteAnnouncer(object):
s = time.time()
address_parts = self.getAddressParts(tracker)
if not address_parts:
- self.site.log.warning("Tracker %s error: Invalid address" % tracker)
+ self.log.warning("Tracker %s error: Invalid address" % tracker)
return False
if tracker not in self.stats:
@@ -188,7 +188,7 @@ class SiteAnnouncer(object):
self.stats[tracker]["time_request"] = time.time()
global_stats[tracker]["time_request"] = time.time()
if config.verbose:
- self.site.log.debug("Tracker announcing to %s (mode: %s)" % (tracker, mode))
+ self.log.debug("Tracker announcing to %s (mode: %s)" % (tracker, mode))
if mode == "update":
num_want = 10
else:
@@ -202,7 +202,7 @@ class SiteAnnouncer(object):
else:
raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"])
except Exception as err:
- self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
+ self.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
error = err
if error:
@@ -210,11 +210,11 @@ class SiteAnnouncer(object):
self.stats[tracker]["time_status"] = time.time()
self.stats[tracker]["last_error"] = str(error)
self.stats[tracker]["time_last_error"] = time.time()
- if self.site.connection_server.has_internet:
+ if self.connection_server.has_internet:
self.stats[tracker]["num_error"] += 1
self.stats[tracker]["num_request"] += 1
global_stats[tracker]["num_request"] += 1
- if self.site.connection_server.has_internet:
+ if self.connection_server.has_internet:
global_stats[tracker]["num_error"] += 1
self.updateWebsocket(tracker="error")
return False
@@ -249,39 +249,106 @@ class SiteAnnouncer(object):
self.site.updateWebsocket(peers_added=added)
if config.verbose:
- self.site.log.debug(
+ self.log.debug(
"Tracker result: %s://%s (found %s peers, new: %s, total: %s)" %
(address_parts["protocol"], address_parts["address"], len(peers), added, len(self.site.peers))
)
return time.time() - s
- @util.Noparallel(blocking=False)
- def announcePex(self, query_num=2, need_num=5):
- peers = self.site.getConnectedPeers()
- if len(peers) == 0: # Wait 3s for connections
- time.sleep(3)
- peers = self.site.getConnectedPeers()
+ def announceToTrackers(self, trackers, force=False, mode="start"):
+ errors = []
+ slow = []
+ s = time.time()
+ threads = []
+ num_announced = 0
- if len(peers) == 0: # Small number of connected peers for this site, connect to any
- peers = list(self.site.getRecentPeers(20))
- need_num = 10
+ for tracker in trackers: # Start announce threads
+ thread = self.site.greenlet_manager.spawn(self.announceTracker, tracker, mode=mode)
+ threads.append(thread)
+ thread.tracker = tracker
- random.shuffle(peers)
- done = 0
- total_added = 0
- for peer in peers:
- num_added = peer.pex(need_num=need_num)
- if num_added is not False:
- done += 1
- total_added += num_added
- if num_added:
- self.site.worker_manager.onPeers()
- self.site.updateWebsocket(peers_added=num_added)
+ time.sleep(0.01)
+ self.updateWebsocket(trackers="announcing")
+
+ gevent.joinall(threads, timeout=20) # Wait for announce finish
+
+ for thread in threads:
+ if thread.value is None:
+ continue
+ if thread.value is not False:
+ if thread.value > 1.0: # Takes more than 1 second to announce
+ slow.append("%.2fs %s" % (thread.value, thread.tracker))
+ num_announced += 1
else:
+ if thread.ready():
+ errors.append(thread.tracker)
+ else: # Still running
+ slow.append("30s+ %s" % thread.tracker)
+
+ # Save peers num
+ self.site.settings["peers"] = len(self.site.peers)
+
+ if len(errors) < len(threads): # At least one tracker finished
+ if len(trackers) == 1:
+ announced_to = trackers[0]
+ else:
+ announced_to = "%s/%s trackers" % (num_announced, len(threads))
+ if mode != "update" or config.verbose:
+ self.log.debug(
+ "Announced in mode %s to %s in %.3fs, errors: %s, slow: %s" %
+ (mode, announced_to, time.time() - s, errors, slow)
+ )
+ else:
+ if len(threads) > 1:
+ self.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s))
+ if len(threads) > 1 and mode != "start": # Move to next tracker
+ self.log.debug("Tracker failed, skipping to next one...")
+ self.site.greenlet_manager.spawnLater(5.0, self.announce, force=force, mode=mode, pex=False)
+
+ self.updateWebsocket(trackers="announced")
+
+ @util.Noparallel(blocking=False)
+ def announcePex(self, query_num=2, need_num=10, establish_connections=True):
+ peers = []
+ try:
+ peer_count = 20 + query_num * 2
+
+ # Wait for some peers to connect
+ for _ in range(5):
+ if not self.site.isServing():
+ return
+ peers = self.site.getConnectedPeers(only_fully_connected=True)
+ if len(peers) > 0:
+ break
+ time.sleep(2)
+
+ if len(peers) < peer_count and establish_connections:
+ # Small number of connected peers for this site, connect to any
+ peers = list(self.site.getRecentPeers(peer_count))
+
+ if len(peers) > 0:
+ self.updateWebsocket(pex="announcing")
+
+ random.shuffle(peers)
+ done = 0
+ total_added = 0
+ for peer in peers:
+ if not establish_connections and not peer.isConnected():
+ continue
+ num_added = peer.pex(need_num=need_num)
+ if num_added is not False:
+ done += 1
+ total_added += num_added
+ if num_added:
+ self.site.worker_manager.onPeers()
+ self.site.updateWebsocket(peers_added=num_added)
+ if done == query_num:
+ break
time.sleep(0.1)
- if done == query_num:
- break
- self.site.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added))
+ self.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added))
+ finally:
+ if len(peers) > 0:
+ self.updateWebsocket(pex="announced")
def updateWebsocket(self, **kwargs):
if kwargs:
diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py
new file mode 100644
index 00000000..90a298cf
--- /dev/null
+++ b/src/Site/SiteHelpers.py
@@ -0,0 +1,256 @@
+import time
+import weakref
+import gevent
+
+class ConnectRequirement(object):
+ next_id = 1
+ def __init__(self, need_nr_peers, need_nr_connected_peers, expiration_interval=None):
+ self.need_nr_peers = need_nr_peers # how many total peers we need
+ self.need_nr_connected_peers = need_nr_connected_peers # how many connected peers we need
+ self.result = gevent.event.AsyncResult() # resolves on need_nr_peers condition
+ self.result_connected = gevent.event.AsyncResult() # resolves on need_nr_connected_peers condition
+
+ self.expiration_interval = expiration_interval
+ self.expired = False
+ if expiration_interval:
+ self.expire_at = time.time() + expiration_interval
+ else:
+ self.expire_at = None
+
+ self.nr_peers = -1 # updated PeerConnector()
+ self.nr_connected_peers = -1 # updated PeerConnector()
+
+ self.heartbeat = gevent.event.AsyncResult()
+
+ self.id = type(self).next_id
+ type(self).next_id += 1
+
+ def fulfilled(self):
+ return self.result.ready() and self.result_connected.ready()
+
+ def ready(self):
+ return self.expired or self.fulfilled()
+
+ # Heartbeat sent when any of the following happens:
+ # * self.result is set
+ # * self.result_connected is set
+ # * self.nr_peers changed
+ # * self.nr_peers_connected changed
+ # * self.expired is set
+ def waitHeartbeat(self, timeout=None):
+ if self.heartbeat.ready():
+ self.heartbeat = gevent.event.AsyncResult()
+ return self.heartbeat.wait(timeout=timeout)
+
+ def sendHeartbeat(self):
+ self.heartbeat.set_result()
+ if self.heartbeat.ready():
+ self.heartbeat = gevent.event.AsyncResult()
+
+class PeerConnector(object):
+
+ def __init__(self, site):
+ self.site = site
+
+ self.peer_reqs = weakref.WeakValueDictionary() # How many connected peers we need.
+ # Separate entry for each requirement.
+ # Objects of type ConnectRequirement.
+ self.peer_connector_controller = None # Thread doing the orchestration in background.
+ self.peer_connector_workers = dict() # Threads trying to connect to individual peers.
+ self.peer_connector_worker_limit = 5 # Max nr of workers.
+ self.peer_connector_announcer = None # Thread doing announces in background.
+
+ # Max effective values. Set by processReqs().
+ self.need_nr_peers = 0
+ self.need_nr_connected_peers = 0
+ self.nr_peers = 0 # set by processReqs()
+ self.nr_connected_peers = 0 # set by processReqs2()
+
+ # Connector Controller state
+ self.peers = list()
+
+ def addReq(self, req):
+ self.peer_reqs[req.id] = req
+ self.processReqs()
+
+ def newReq(self, need_nr_peers, need_nr_connected_peers, expiration_interval=None):
+ req = ConnectRequirement(need_nr_peers, need_nr_connected_peers, expiration_interval=expiration_interval)
+ self.addReq(req)
+ return req
+
+ def processReqs(self, nr_connected_peers=None):
+ nr_peers = len(self.site.peers)
+ self.nr_peers = nr_peers
+
+ need_nr_peers = 0
+ need_nr_connected_peers = 0
+
+ items = list(self.peer_reqs.items())
+ for key, req in items:
+ send_heartbeat = False
+
+ if req.expire_at and req.expire_at < time.time():
+ req.expired = True
+ self.peer_reqs.pop(key, None)
+ send_heartbeat = True
+ elif req.result.ready() and req.result_connected.ready():
+ pass
+ else:
+ if nr_connected_peers is not None:
+ if req.need_nr_peers <= nr_peers and req.need_nr_connected_peers <= nr_connected_peers:
+ req.result.set_result(nr_peers)
+ req.result_connected.set_result(nr_connected_peers)
+ send_heartbeat = True
+ if req.nr_peers != nr_peers or req.nr_connected_peers != nr_connected_peers:
+ req.nr_peers = nr_peers
+ req.nr_connected_peers = nr_connected_peers
+ send_heartbeat = True
+
+ if not (req.result.ready() and req.result_connected.ready()):
+ need_nr_peers = max(need_nr_peers, req.need_nr_peers)
+ need_nr_connected_peers = max(need_nr_connected_peers, req.need_nr_connected_peers)
+
+ if send_heartbeat:
+ req.sendHeartbeat()
+
+ self.need_nr_peers = need_nr_peers
+ self.need_nr_connected_peers = need_nr_connected_peers
+
+ if nr_connected_peers is None:
+ nr_connected_peers = 0
+ if need_nr_peers > nr_peers:
+ self.spawnPeerConnectorAnnouncer();
+ if need_nr_connected_peers > nr_connected_peers:
+ self.spawnPeerConnectorController();
+
+ def processReqs2(self):
+ self.nr_connected_peers = len(self.site.getConnectedPeers(only_fully_connected=True))
+ self.processReqs(nr_connected_peers=self.nr_connected_peers)
+
+ # For adding new peers when ConnectorController is working.
+ # While it is iterating over a cached list of peers, there can be a significant lag
+ # for a newly discovered peer to get in sight of the controller.
+ # Suppose most previously known peers are dead and we've just get a few
+ # new peers from a tracker.
+ # So we mix the new peer to the cached list.
+ # When ConnectorController is stopped (self.peers is empty), we just do nothing here.
+ def addPeer(self, peer):
+ if not self.peers:
+ return
+ if peer not in self.peers:
+ self.peers.append(peer)
+
+ def deregisterPeer(self, peer):
+ try:
+ self.peers.remove(peer)
+ except:
+ pass
+
+ def sleep(self, t):
+ self.site.connection_server.sleep(t)
+
+ def keepGoing(self):
+ return self.site.isServing() and self.site.connection_server.allowsCreatingConnections()
+
+ def peerConnectorWorker(self, peer):
+ if not peer.isConnected():
+ peer.connect()
+ if peer.isConnected():
+ peer.ping()
+ self.processReqs2()
+
+ def peerConnectorController(self):
+ self.peers = list()
+ addendum = 20
+ while self.keepGoing():
+
+ no_peers_loop = 0
+ while len(self.site.peers) < 1:
+ # No peers at all.
+ # Waiting for the announcer to discover some peers.
+ self.sleep(10 + no_peers_loop)
+ no_peers_loop += 1
+ if not self.keepGoing() or no_peers_loop > 60:
+ break
+
+ self.processReqs2()
+
+ if self.need_nr_connected_peers <= self.nr_connected_peers:
+ # Ok, nobody waits for connected peers.
+ # Done.
+ break
+
+ if len(self.site.peers) < 1:
+ break
+
+ if len(self.peers) < 1:
+ # refill the peer list
+ self.peers = self.site.getRecentPeers(self.need_nr_connected_peers * 2 + self.nr_connected_peers + addendum)
+ addendum = min(addendum * 2 + 50, 10000)
+ if len(self.peers) <= self.nr_connected_peers:
+ # Looks like all known peers are connected.
+ # Waiting for the announcer to discover some peers.
+ self.site.announcer.announcePex(establish_connections=False)
+ self.sleep(10)
+ continue
+
+ added = 0
+
+ # try connecting to peers
+ while self.keepGoing() and len(self.peer_connector_workers) < self.peer_connector_worker_limit:
+ if len(self.peers) < 1:
+ break
+
+ peer = self.peers.pop(0)
+
+ if peer.isConnected():
+ continue
+
+ thread = self.peer_connector_workers.get(peer, None)
+ if thread:
+ continue
+
+ thread = self.site.spawn(self.peerConnectorWorker, peer)
+ self.peer_connector_workers[peer] = thread
+ thread.link(lambda thread, peer=peer: self.peer_connector_workers.pop(peer, None))
+ added += 1
+
+ if not self.keepGoing():
+ break
+
+ if not added:
+ # Looks like all known peers are either connected or being connected,
+ # so we weren't able to start connecting any peer in this iteration.
+ # Waiting for the announcer to discover some peers.
+ self.sleep(20)
+
+ # wait for more room in self.peer_connector_workers
+ while self.keepGoing() and len(self.peer_connector_workers) >= self.peer_connector_worker_limit:
+ self.sleep(2)
+
+ if not self.site.connection_server.isInternetOnline():
+ self.sleep(30)
+
+ self.peers = list()
+ self.peer_connector_controller = None
+
+ def peerConnectorAnnouncer(self):
+ while self.keepGoing():
+ if self.need_nr_peers <= self.nr_peers:
+ break
+ self.site.announce(mode="more")
+ self.processReqs2()
+ if self.need_nr_peers <= self.nr_peers:
+ break
+ self.sleep(10)
+ if not self.site.connection_server.isInternetOnline():
+ self.sleep(20)
+ self.peer_connector_announcer = None
+
+ def spawnPeerConnectorController(self):
+ if self.peer_connector_controller is None or self.peer_connector_controller.ready():
+ self.peer_connector_controller = self.site.spawn(self.peerConnectorController)
+
+ def spawnPeerConnectorAnnouncer(self):
+ if self.peer_connector_announcer is None or self.peer_connector_announcer.ready():
+ self.peer_connector_announcer = self.site.spawn(self.peerConnectorAnnouncer)
diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py
index 684d69fc..8175a1f5 100644
--- a/src/Site/SiteManager.py
+++ b/src/Site/SiteManager.py
@@ -4,6 +4,7 @@ import re
import os
import time
import atexit
+import collections
import gevent
@@ -27,6 +28,21 @@ class SiteManager(object):
gevent.spawn(self.saveTimer)
atexit.register(lambda: self.save(recalculate_size=True))
+ # ZeroNet has a bug of desyncing between:
+ # * time sent in a response of listModified
+ # and
+ # * time checked on receiving a file.
+ # This leads to the following scenario:
+ # * Request listModified.
+ # * Detect that the remote peer missing an update
+ # * Send a newer version of the file back to the peer.
+ # * The peer responses "ok: File not changed"
+ # .....
+ # * Request listModified the next time and do all the same again.
+ # So we keep the list of sent back entries to prevent sending multiple useless updates:
+ # "{site.address} - {peer.key} - {inner_path}" -> mtime
+ self.send_back_lru = collections.OrderedDict()
+
# Load all sites from data/sites.json
@util.Noparallel()
def load(self, cleanup=True, startup=False):
@@ -155,6 +171,11 @@ class SiteManager(object):
def resolveDomainCached(self, domain):
return self.resolveDomain(domain)
+ # Checks if the address is blocked. To be implemented in content filter plugins.
+ # Since 0.8.0
+ def isAddressBlocked(self, address):
+ return False
+
# Return: Site object or None if not found
def get(self, address):
if self.isDomainCached(address):
@@ -216,6 +237,23 @@ class SiteManager(object):
self.load(startup=True)
return self.sites
+ # Return False if we never sent to
+ # or if the file that was sent was older than
+ # so that send back logic is suppressed for .
+ # True if can be sent back to .
+ def checkSendBackLRU(self, site, peer, inner_path, remote_modified):
+ key = site.address + ' - ' + peer.key + ' - ' + inner_path
+ sent_modified = self.send_back_lru.get(key, 0)
+ return remote_modified < sent_modified
+
+ def addToSendBackLRU(self, site, peer, inner_path, modified):
+ key = site.address + ' - ' + peer.key + ' - ' + inner_path
+ if self.send_back_lru.get(key, None) is None:
+ self.send_back_lru[key] = modified
+ while len(self.send_back_lru) > config.send_back_lru_size:
+ self.send_back_lru.popitem(last=False)
+ else:
+ self.send_back_lru.move_to_end(key, last=True)
site_manager = SiteManager() # Singletone
diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py
index 27032e79..d5df8e95 100644
--- a/src/Site/SiteStorage.py
+++ b/src/Site/SiteStorage.py
@@ -24,6 +24,25 @@ thread_pool_fs_read = ThreadPool.ThreadPool(config.threads_fs_read, name="FS rea
thread_pool_fs_write = ThreadPool.ThreadPool(config.threads_fs_write, name="FS write")
thread_pool_fs_batch = ThreadPool.ThreadPool(1, name="FS batch")
+class VerifyFiles_Notificator(object):
+ def __init__(self, site, quick_check):
+ self.site = site
+ self.quick_check = quick_check
+ self.scanned_files = 0
+ self.websocket_update_interval = 0.25
+ self.websocket_update_time = time.time()
+
+ def inc(self):
+ self.scanned_files += 1
+ if self.websocket_update_time + self.websocket_update_interval < time.time():
+ self.send()
+
+ def send(self):
+ self.websocket_update_time = time.time()
+ if self.quick_check:
+ self.site.updateWebsocket(checking=self.scanned_files)
+ else:
+ self.site.updateWebsocket(verifying=self.scanned_files)
@PluginManager.acceptPlugins
class SiteStorage(object):
@@ -260,7 +279,7 @@ class SiteStorage(object):
# Open file object
@thread_pool_fs_read.wrap
def read(self, inner_path, mode="rb"):
- return self.open(inner_path, mode).read()
+ return open(self.getPath(inner_path), mode).read()
@thread_pool_fs_write.wrap
def writeThread(self, inner_path, content):
@@ -356,7 +375,7 @@ class SiteStorage(object):
# Reopen DB to check changes
if self.has_db:
self.closeDb("New dbschema")
- gevent.spawn(self.getDb)
+ self.site.spawn(self.getDb)
elif not config.disable_db and should_load_to_db and self.has_db: # Load json file to db
if config.verbose:
self.log.debug("Loading json file to db: %s (file: %s)" % (inner_path, file))
@@ -369,11 +388,11 @@ class SiteStorage(object):
# Load and parse json file
@thread_pool_fs_read.wrap
def loadJson(self, inner_path):
- try:
- with self.open(inner_path, "r", encoding="utf8") as file:
+ try :
+ with self.open(inner_path) as file:
return json.load(file)
except Exception as err:
- self.log.warning("Json load error: %s" % Debug.formatException(err))
+ self.log.error("Json load error: %s" % Debug.formatException(err))
return None
# Write formatted json file
@@ -424,6 +443,8 @@ class SiteStorage(object):
return inner_path
# Verify all files sha512sum using content.json
+ # The result may not be accurate if self.site.isStopping().
+ # verifyFiles() return immediately in that case.
def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = []
back = defaultdict(int)
@@ -435,17 +456,55 @@ class SiteStorage(object):
self.log.debug("VerifyFile content.json not exists")
self.site.needFile("content.json", update=True) # Force update to fix corrupt file
self.site.content_manager.loadContent() # Reload content.json
- for content_inner_path, content in list(self.site.content_manager.contents.items()):
+
+ # Trying to read self.site.content_manager.contents without being stuck
+ # on reading the long file list and also without getting
+ # "RuntimeError: dictionary changed size during iteration"
+ # We can't use just list(iteritems()) since it loads all the contents files
+ # at once and gets unresponsive.
+ contents = {}
+ notificator = None
+ tries = 0
+ max_tries = 40
+ stop = False
+ while not stop:
+ try:
+ contents = {}
+ notificator = VerifyFiles_Notificator(self.site, quick_check)
+ for content_inner_path, content in self.site.content_manager.contents.iteritems():
+ notificator.inc()
+ contents[content_inner_path] = content
+ if self.site.isStopping():
+ stop = True
+ break
+ stop = True
+ except RuntimeError as err:
+ if "changed size during iteration" in str(err):
+ tries += 1
+ if tries >= max_tries:
+ self.log.info("contents.json file list changed during iteration. %s tries done. Giving up.", tries)
+ stop = True
+ self.log.info("contents.json file list changed during iteration. Trying again... (%s)", tries)
+ time.sleep(2 * tries)
+ else:
+ stop = True
+
+ for content_inner_path, content in contents.items():
back["num_content"] += 1
i += 1
if i % 50 == 0:
time.sleep(0.001) # Context switch to avoid gevent hangs
+
+ if self.site.isStopping():
+ break
+
if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file
back["num_content_missing"] += 1
self.log.debug("[MISSING] %s" % content_inner_path)
bad_files.append(content_inner_path)
for file_relative_path in list(content.get("files", {}).keys()):
+ notificator.inc()
back["num_file"] += 1
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
@@ -456,15 +515,19 @@ class SiteStorage(object):
bad_files.append(file_inner_path)
continue
+ err = None
+
if quick_check:
- ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"]
+ file_size = os.path.getsize(file_path)
+ expected_size = content["files"][file_relative_path]["size"]
+ ok = file_size == expected_size
if not ok:
- err = "Invalid size"
+ err = "Invalid size: %s - actual, %s - expected" % (file_size, expected_size)
else:
try:
ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))
- except Exception as _err:
- err = _err
+ except Exception as err2:
+ err = err2
ok = False
if not ok:
@@ -477,6 +540,7 @@ class SiteStorage(object):
optional_added = 0
optional_removed = 0
for file_relative_path in list(content.get("files_optional", {}).keys()):
+ notificator.inc()
back["num_optional"] += 1
file_node = content["files_optional"][file_relative_path]
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
@@ -521,6 +585,8 @@ class SiteStorage(object):
(content_inner_path, len(content["files"]), quick_check, optional_added, optional_removed)
)
+ notificator.send()
+
self.site.content_manager.contents.db.processDelayed()
time.sleep(0.001) # Context switch to avoid gevent hangs
return back
diff --git a/src/Test/TestFileRequest.py b/src/Test/TestFileRequest.py
index 3fabc271..51dad600 100644
--- a/src/Test/TestFileRequest.py
+++ b/src/Test/TestFileRequest.py
@@ -16,7 +16,7 @@ class TestFileRequest:
client = ConnectionServer(file_server.ip, 1545)
connection = client.getConnection(file_server.ip, 1544)
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Normal request
response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0})
@@ -61,7 +61,7 @@ class TestFileRequest:
file_server.ip_incoming = {} # Reset flood protection
client = ConnectionServer(file_server.ip, 1545)
connection = client.getConnection(file_server.ip, 1544)
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
buff = io.BytesIO()
response = connection.request("streamFile", {"site": site.address, "inner_path": "content.json", "location": 0}, buff)
@@ -89,7 +89,7 @@ class TestFileRequest:
client.stop()
def testPex(self, file_server, site, site_temp):
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
client = FileServer(file_server.ip, 1545)
client.sites = {site_temp.address: site_temp}
site_temp.connection_server = client
diff --git a/src/Test/TestPeer.py b/src/Test/TestPeer.py
index f57e046e..2d587ef4 100644
--- a/src/Test/TestPeer.py
+++ b/src/Test/TestPeer.py
@@ -13,7 +13,7 @@ from . import Spy
@pytest.mark.usefixtures("resetTempSettings")
class TestPeer:
def testPing(self, file_server, site, site_temp):
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
client = FileServer(file_server.ip, 1545)
client.sites = {site_temp.address: site_temp}
site_temp.connection_server = client
@@ -32,7 +32,7 @@ class TestPeer:
client.stop()
def testDownloadFile(self, file_server, site, site_temp):
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
client = FileServer(file_server.ip, 1545)
client.sites = {site_temp.address: site_temp}
site_temp.connection_server = client
@@ -77,11 +77,11 @@ class TestPeer:
def testHashfieldExchange(self, file_server, site, site_temp):
server1 = file_server
- server1.sites[site.address] = site
+ server1.getSites()[site.address] = site
site.connection_server = server1
server2 = FileServer(file_server.ip, 1545)
- server2.sites[site_temp.address] = site_temp
+ server2.getSites()[site_temp.address] = site_temp
site_temp.connection_server = server2
site.storage.verifyFiles(quick_check=True) # Find what optional files we have
@@ -127,7 +127,7 @@ class TestPeer:
server2.stop()
def testFindHash(self, file_server, site, site_temp):
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
client = FileServer(file_server.ip, 1545)
client.sites = {site_temp.address: site_temp}
site_temp.connection_server = client
diff --git a/src/Test/TestSiteDownload.py b/src/Test/TestSiteDownload.py
index cd0a4c9f..37fe796b 100644
--- a/src/Test/TestSiteDownload.py
+++ b/src/Test/TestSiteDownload.py
@@ -23,7 +23,7 @@ class TestSiteDownload:
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -74,7 +74,7 @@ class TestSiteDownload:
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -130,7 +130,7 @@ class TestSiteDownload:
def testArchivedDownload(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -178,7 +178,7 @@ class TestSiteDownload:
def testArchivedBeforeDownload(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -229,7 +229,7 @@ class TestSiteDownload:
def testOptionalDownload(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = ConnectionServer(file_server.ip, 1545)
@@ -271,7 +271,7 @@ class TestSiteDownload:
def testFindOptional(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init full source server (has optional files)
site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
@@ -284,7 +284,7 @@ class TestSiteDownload:
gevent.spawn(listen)
time.sleep(0.001) # Port opening
- file_server_full.sites[site_full.address] = site_full # Add site
+ file_server_full.getSites()[site_full.address] = site_full # Add site
site_full.storage.verifyFiles(quick_check=True) # Check optional files
site_full_peer = site.addPeer(file_server.ip, 1546) # Add it to source server
hashfield = site_full_peer.updateHashfield() # Update hashfield
@@ -342,7 +342,7 @@ class TestSiteDownload:
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -423,7 +423,7 @@ class TestSiteDownload:
def testBigUpdate(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -476,7 +476,7 @@ class TestSiteDownload:
def testHugeContentSiteUpdate(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
@@ -524,7 +524,7 @@ class TestSiteDownload:
# Init source server
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Init client server
client = FileServer(file_server.ip, 1545)
diff --git a/src/Test/TestTor.py b/src/Test/TestTor.py
index e6b82c1a..120cd53e 100644
--- a/src/Test/TestTor.py
+++ b/src/Test/TestTor.py
@@ -75,7 +75,7 @@ class TestTor:
assert file_server.getConnection(address + ".onion", 1544, site=site) != file_server.getConnection(address + ".onion", 1544, site=site_temp)
# Only allow to query from the locked site
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
connection_locked = file_server.getConnection(address + ".onion", 1544, site=site)
assert "body" in connection_locked.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0})
assert connection_locked.request("getFile", {"site": "1OTHERSITE", "inner_path": "content.json", "location": 0})["error"] == "Invalid site"
@@ -83,11 +83,11 @@ class TestTor:
def testPex(self, file_server, site, site_temp):
# Register site to currently running fileserver
site.connection_server = file_server
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
# Create a new file server to emulate new peer connecting to our peer
file_server_temp = FileServer(file_server.ip, 1545)
site_temp.connection_server = file_server_temp
- file_server_temp.sites[site_temp.address] = site_temp
+ file_server_temp.getSites()[site_temp.address] = site_temp
# We will request peers from this
peer_source = site_temp.addPeer(file_server.ip, 1544)
@@ -113,7 +113,7 @@ class TestTor:
def testFindHash(self, tor_manager, file_server, site, site_temp):
file_server.ip_incoming = {} # Reset flood protection
- file_server.sites[site.address] = site
+ file_server.getSites()[site.address] = site
file_server.tor_manager = tor_manager
client = FileServer(file_server.ip, 1545)
diff --git a/src/Test/testdata/chart.db-shm b/src/Test/testdata/chart.db-shm
new file mode 100644
index 00000000..7c822be1
Binary files /dev/null and b/src/Test/testdata/chart.db-shm differ
diff --git a/src/Test/testdata/chart.db-wal b/src/Test/testdata/chart.db-wal
new file mode 100644
index 00000000..dbb007b0
Binary files /dev/null and b/src/Test/testdata/chart.db-wal differ
diff --git a/src/Test/testdata/content.db-shm b/src/Test/testdata/content.db-shm
new file mode 100644
index 00000000..dfcb5880
Binary files /dev/null and b/src/Test/testdata/content.db-shm differ
diff --git a/src/Test/testdata/content.db-wal b/src/Test/testdata/content.db-wal
new file mode 100644
index 00000000..a7892237
Binary files /dev/null and b/src/Test/testdata/content.db-wal differ
diff --git a/src/Test/testdata/filters.json b/src/Test/testdata/filters.json
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/Test/testdata/filters.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/Test/testdata/openssl.cnf b/src/Test/testdata/openssl.cnf
new file mode 100644
index 00000000..ff59f6a4
--- /dev/null
+++ b/src/Test/testdata/openssl.cnf
@@ -0,0 +1,58 @@
+[ req ]
+default_bits = 2048
+default_keyfile = server-key.pem
+distinguished_name = subject
+req_extensions = req_ext
+x509_extensions = x509_ext
+string_mask = utf8only
+
+# The Subject DN can be formed using X501 or RFC 4514 (see RFC 4519 for a description).
+# Its sort of a mashup. For example, RFC 4514 does not provide emailAddress.
+[ subject ]
+countryName = US
+stateOrProvinceName = NY
+localityName = New York
+organizationName = Example, LLC
+
+# Use a friendly name here because its presented to the user. The server's DNS
+# names are placed in Subject Alternate Names. Plus, DNS names here is deprecated
+# by both IETF and CA/Browser Forums. If you place a DNS name here, then you
+# must include the DNS name in the SAN too (otherwise, Chrome and others that
+# strictly follow the CA/Browser Baseline Requirements will fail).
+commonName = Example Company
+
+emailAddress = test@example.com
+
+# Section x509_ext is used when generating a self-signed certificate. I.e., openssl req -x509 ...
+[ x509_ext ]
+
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth, serverAuth
+subjectAltName = @alternate_names
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+# extendedKeyUsage = serverAuth, clientAuth
+
+# Section req_ext is used when generating a certificate signing request. I.e., openssl req ...
+[ req_ext ]
+
+subjectKeyIdentifier = hash
+
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth, serverAuth
+subjectAltName = @alternate_names
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+# extendedKeyUsage = serverAuth, clientAuth
+
+[ alternate_names ]
+
+DNS.1 = nazwa.pl
+DNS.2 = www.nazwa.pl
\ No newline at end of file
diff --git a/src/Test/testdata/sites.json b/src/Test/testdata/sites.json
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/Test/testdata/sites.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/Test/testdata/trackers.json b/src/Test/testdata/trackers.json
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/Test/testdata/trackers.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/Test/testdata/users.json b/src/Test/testdata/users.json
new file mode 100644
index 00000000..fdadced4
--- /dev/null
+++ b/src/Test/testdata/users.json
@@ -0,0 +1,9 @@
+
+ {
+ "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc": {
+ "certs": {},
+ "master_seed": "024bceac1105483d66585d8a60eaf20aa8c3254b0f266e0d626ddb6114e2949a",
+ "sites": {}
+ }
+ }
+
\ No newline at end of file
diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py
index 4a4e0545..dbd3ca67 100644
--- a/src/Ui/UiRequest.py
+++ b/src/Ui/UiRequest.py
@@ -543,37 +543,18 @@ class UiRequest(object):
if show_loadingscreen:
meta_tags += '';
-
- def xescape(s):
- '''combines parts from re.escape & html.escape'''
- # https://github.com/python/cpython/blob/3.10/Lib/re.py#L267
- # '&' is handled otherwise
- re_chars = {i: '\\' + chr(i) for i in b'()[]{}*+-|^$\\.~# \t\n\r\v\f'}
- # https://github.com/python/cpython/blob/3.10/Lib/html/__init__.py#L12
- html_chars = {
- '<' : '<',
- '>' : '>',
- '"' : '"',
- "'" : ''',
- }
- # we can't replace '&' because it makes certain zites work incorrectly
- # it should however in no way interfere with re.sub in render
- repl = {}
- repl.update(re_chars)
- repl.update(html_chars)
- return s.translate(repl)
-
+
return self.render(
"src/Ui/template/wrapper.html",
server_url=server_url,
inner_path=inner_path,
- file_url=xescape(file_url),
- file_inner_path=xescape(file_inner_path),
+ file_url=re.escape(file_url),
+ file_inner_path=re.escape(file_inner_path),
address=site.address,
- title=xescape(title),
+ title=html.escape(title),
body_style=body_style,
meta_tags=meta_tags,
- query_string=xescape(inner_query_string),
+ query_string=re.escape(inner_query_string),
wrapper_key=site.settings["wrapper_key"],
ajax_key=site.settings["ajax_key"],
wrapper_nonce=wrapper_nonce,
@@ -749,10 +730,7 @@ class UiRequest(object):
def replaceHtmlVariables(self, block, path_parts):
user = self.getCurrentUser()
- if user and user.settings:
- themeclass = "theme-%-6s" % re.sub("[^a-z]", "", user.settings.get("theme", "light"))
- else:
- themeclass = "theme-light"
+ themeclass = "theme-%-6s" % re.sub("[^a-z]", "", user.settings.get("theme", "light"))
block = block.replace(b"{themeclass}", themeclass.encode("utf8"))
if path_parts:
diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py
index 61943ada..bda1daa0 100644
--- a/src/Ui/UiServer.py
+++ b/src/Ui/UiServer.py
@@ -167,7 +167,7 @@ class UiServer:
self.log.error("Web interface bind error, must be running already, exiting.... %s" % err)
import main
main.file_server.stop()
- self.log.debug("Stopped.")
+ self.log.info("Stopped.")
def stop(self):
self.log.debug("Stopping...")
diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py
index 2f982e1d..60d746db 100644
--- a/src/Ui/UiWebsocket.py
+++ b/src/Ui/UiWebsocket.py
@@ -318,6 +318,7 @@ class UiWebsocket(object):
back["updatesite"] = config.updatesite
back["dist_type"] = config.dist_type
back["lib_verify_best"] = CryptBitcoin.lib_verify_best
+ back["passive_mode"] = file_server.passive_mode
return back
def formatAnnouncerInfo(self, site):
@@ -327,10 +328,7 @@ class UiWebsocket(object):
def actionAs(self, to, address, cmd, params=[]):
if not self.hasSitePermission(address, cmd=cmd):
- #TODO! Return this as error ?
return self.response(to, "No permission for site %s" % address)
- if not self.server.sites.get(address):
- return self.response(to, {"error": "Site Does Not Exist: %s" % address})
req_self = copy.copy(self)
req_self.site = self.server.sites.get(address)
req_self.hasCmdPermission = self.hasCmdPermission # Use the same permissions as current site
@@ -422,15 +420,10 @@ class UiWebsocket(object):
is_user_content = file_info and ("cert_signers" in file_info or "cert_signers_pattern" in file_info)
if is_user_content and privatekey is None:
cert = self.user.getCert(self.site.address)
- if not cert:
- error = "Site sign failed: No certificate selected for Site: %s, Hence Signing inner_path: %s Failed, Try Adding/Selecting User Cert via Site Login" % (self.site.address, inner_path)
- self.log.error(error)
- return self.response(to, {"error": error})
- else:
- extend["cert_auth_type"] = cert["auth_type"]
- extend["cert_user_id"] = self.user.getCertUserId(site.address)
- extend["cert_sign"] = cert["cert_sign"]
- self.log.debug("Extending content.json with cert %s" % extend["cert_user_id"])
+ extend["cert_auth_type"] = cert["auth_type"]
+ extend["cert_user_id"] = self.user.getCertUserId(site.address)
+ extend["cert_sign"] = cert["cert_sign"]
+ self.log.debug("Extending content.json with cert %s" % extend["cert_user_id"])
if not self.hasFilePermission(inner_path):
self.log.error("SiteSign error: you don't own this site & site owner doesn't allow you to do so.")
@@ -920,9 +913,9 @@ class UiWebsocket(object):
self.response(to, "ok")
# Update site content.json
- def actionSiteUpdate(self, to, address, check_files=False, since=None, announce=False):
+ def actionSiteUpdate(self, to, address, check_files=False, verify_files=False, since=None, announce=False):
def updateThread():
- site.update(announce=announce, check_files=check_files, since=since)
+ site.update(announce=announce, check_files=check_files, verify_files=verify_files, since=since)
self.response(to, "Updated")
site = self.server.sites.get(address)
@@ -1172,6 +1165,32 @@ class UiWebsocket(object):
file_server.portCheck()
self.response(to, file_server.port_opened)
+ @flag.admin
+ @flag.no_multiuser
+ def actionServerSetPassiveMode(self, to, passive_mode=False):
+ import main
+ file_server = main.file_server
+ if file_server.isPassiveMode() != passive_mode:
+ file_server.setPassiveMode(passive_mode)
+ if file_server.isPassiveMode():
+ self.cmd("notification", ["info", _["Passive mode enabled"], 5000])
+ else:
+ self.cmd("notification", ["info", _["Passive mode disabled"], 5000])
+ self.server.updateWebsocket()
+
+ @flag.admin
+ @flag.no_multiuser
+ def actionServerSetOfflineMode(self, to, offline_mode=False):
+ import main
+ file_server = main.file_server
+ if file_server.isOfflineMode() != offline_mode:
+ file_server.setOfflineMode(offline_mode)
+ if file_server.isOfflineMode():
+ self.cmd("notification", ["info", _["Offline mode enabled"], 5000])
+ else:
+ self.cmd("notification", ["info", _["Offline mode disabled"], 5000])
+ self.server.updateWebsocket()
+
@flag.admin
@flag.no_multiuser
def actionServerShutdown(self, to, restart=False):
@@ -1182,7 +1201,7 @@ class UiWebsocket(object):
return False
if restart:
main.restart_after_shutdown = True
- main.file_server.stop()
+ main.file_server.stop(ui_websocket=self)
main.ui_server.stop()
if restart:
diff --git a/src/loglevel_overrides.py b/src/loglevel_overrides.py
new file mode 100644
index 00000000..5622e523
--- /dev/null
+++ b/src/loglevel_overrides.py
@@ -0,0 +1,9 @@
+# This file is for adding rules for selectively enabling debug logging
+# when working on the code.
+# Add your rules here and skip this file when committing changes.
+
+#import re
+#from util import SelectiveLogger
+#
+#SelectiveLogger.addLogLevelRaisingRule("ConnServer")
+#SelectiveLogger.addLogLevelRaisingRule(re.compile(r'^Site:'))
diff --git a/src/main.py b/src/main.py
index ec90f4d9..21424d47 100644
--- a/src/main.py
+++ b/src/main.py
@@ -4,6 +4,7 @@ import sys
import stat
import time
import logging
+import loglevel_overrides
startup_errors = []
def startupError(msg):
@@ -154,7 +155,7 @@ class Actions(object):
logging.info("Starting servers....")
gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)])
- logging.info("All server stopped")
+ logging.info("All servers stopped")
# Site commands
@@ -254,9 +255,8 @@ class Actions(object):
file_correct = site.content_manager.verifyFile(
content_inner_path, site.storage.open(content_inner_path, "rb"), ignore_same=False
)
- except Exception as exp:
+ except Exception as err:
file_correct = False
- err = exp
if file_correct is True:
logging.info("[OK] %s (Done in %.3fs)" % (content_inner_path, time.time() - s))
diff --git a/src/util/CircularIterator.py b/src/util/CircularIterator.py
new file mode 100644
index 00000000..3466092e
--- /dev/null
+++ b/src/util/CircularIterator.py
@@ -0,0 +1,34 @@
+import random
+
+class CircularIterator:
+ def __init__(self):
+ self.successive_count = 0
+ self.last_size = 0
+ self.index = -1
+
+ def next(self, items):
+ self.last_size = len(items)
+
+ if self.last_size == 0:
+ return None
+
+ if self.index < 0:
+ self.index = random.randint(0, self.last_size)
+ else:
+ self.index += 1
+
+ self.index = self.index % self.last_size
+
+ self.successive_count += 1
+
+ return items[self.index]
+
+ def resetSuccessiveCount(self):
+ self.successive_count = 0
+
+ def getSuccessiveCount(self):
+ return self.successive_count
+
+ def isWrapped(self):
+ return self.successive_count >= self.last_size
+
diff --git a/src/util/Diff.py b/src/util/Diff.py
index 53b82c5a..8281188b 100644
--- a/src/util/Diff.py
+++ b/src/util/Diff.py
@@ -42,8 +42,6 @@ def patch(old_f, actions):
continue
elif action == "+": # Add lines
for add_line in param:
- if type(add_line) is str:
- add_line = add_line.encode()
new_f.write(add_line)
else:
raise "Unknown action: %s" % action
diff --git a/src/util/GreenletManager.py b/src/util/GreenletManager.py
index e024233d..d711d09a 100644
--- a/src/util/GreenletManager.py
+++ b/src/util/GreenletManager.py
@@ -3,17 +3,37 @@ from Debug import Debug
class GreenletManager:
- def __init__(self):
+ # pool is either gevent.pool.Pool or GreenletManager.
+ # if pool is None, new gevent.pool.Pool() is created.
+ def __init__(self, pool=None):
self.greenlets = set()
+ if not pool:
+ pool = gevent.pool.Pool(None)
+ self.pool = pool
+
+ def _spawn_later(self, seconds, *args, **kwargs):
+ # If pool is another GreenletManager, delegate to it.
+ if hasattr(self.pool, 'spawnLater'):
+ return self.pool.spawnLater(seconds, *args, **kwargs)
+
+ # There's gevent.spawn_later(), but there isn't gevent.pool.Pool.spawn_later().
+ # Doing manually.
+ greenlet = self.pool.greenlet_class(*args, **kwargs)
+ self.pool.add(greenlet)
+ greenlet.start_later(seconds)
+ return greenlet
+
+ def _spawn(self, *args, **kwargs):
+ return self.pool.spawn(*args, **kwargs)
def spawnLater(self, *args, **kwargs):
- greenlet = gevent.spawn_later(*args, **kwargs)
+ greenlet = self._spawn_later(*args, **kwargs)
greenlet.link(lambda greenlet: self.greenlets.remove(greenlet))
self.greenlets.add(greenlet)
return greenlet
def spawn(self, *args, **kwargs):
- greenlet = gevent.spawn(*args, **kwargs)
+ greenlet = self._spawn(*args, **kwargs)
greenlet.link(lambda greenlet: self.greenlets.remove(greenlet))
self.greenlets.add(greenlet)
return greenlet
diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py
index 6018e2d3..8c394a84 100644
--- a/src/util/SafeRe.py
+++ b/src/util/SafeRe.py
@@ -1,10 +1,16 @@
import re
+import logging
+
+log = logging.getLogger("SafeRe")
+
class UnsafePatternError(Exception):
pass
+max_cache_size = 1000
cached_patterns = {}
+old_cached_patterns = {}
def isSafePattern(pattern):
@@ -15,18 +21,78 @@ def isSafePattern(pattern):
if unsafe_pattern_match:
raise UnsafePatternError("Potentially unsafe part of the pattern: %s in %s" % (unsafe_pattern_match.group(0), pattern))
- repetitions = re.findall(r"\.[\*\{\+]", pattern)
- if len(repetitions) >= 10:
- raise UnsafePatternError("More than 10 repetitions of %s in %s" % (repetitions[0], pattern))
+ repetitions1 = re.findall(r"\.[\*\{\+]", pattern)
+ repetitions2 = re.findall(r"[^(][?]", pattern)
+ if len(repetitions1) + len(repetitions2) >= 10:
+ raise UnsafePatternError("More than 10 repetitions in %s" % pattern)
return True
-def match(pattern, *args, **kwargs):
+def compilePattern(pattern):
+ global cached_patterns
+ global old_cached_patterns
+
cached_pattern = cached_patterns.get(pattern)
if cached_pattern:
- return cached_pattern.match(*args, **kwargs)
- else:
- if isSafePattern(pattern):
- cached_patterns[pattern] = re.compile(pattern)
- return cached_patterns[pattern].match(*args, **kwargs)
+ return cached_pattern
+
+ cached_pattern = old_cached_patterns.get(pattern)
+ if cached_pattern:
+ del old_cached_patterns[pattern]
+ cached_patterns[pattern] = cached_pattern
+ return cached_pattern
+
+ if isSafePattern(pattern):
+ cached_pattern = re.compile(pattern)
+ cached_patterns[pattern] = cached_pattern
+ log.debug("Compiled new pattern: %s" % pattern)
+ log.debug("Cache size: %d + %d" % (len(cached_patterns), len(old_cached_patterns)))
+
+ if len(cached_patterns) > max_cache_size:
+ old_cached_patterns = cached_patterns
+ cached_patterns = {}
+ log.debug("Size limit reached. Rotating cache.")
+ log.debug("Cache size: %d + %d" % (len(cached_patterns), len(old_cached_patterns)))
+
+ return cached_pattern
+
+
+def match(pattern, *args, **kwargs):
+ cached_pattern = compilePattern(pattern)
+ return cached_pattern.match(*args, **kwargs)
+
+################################################################################
+
+# TESTS
+
+def testSafePattern(pattern):
+ try:
+ return isSafePattern(pattern)
+ except UnsafePatternError as err:
+ return False
+
+
+# Some real examples to make sure it works as expected
+assert testSafePattern('(data/mp4/.*|updater/.*)')
+assert testSafePattern('((js|css)/(?!all.(js|css)))|.git')
+
+
+# Unsafe cases:
+
+# ((?!json).)*$ not allowed, because of ) before the * character. Possible fix: .*(?!json)$
+assert not testSafePattern('((?!json).)*$')
+assert testSafePattern('.*(?!json)$')
+
+# (.*.epub|.*.jpg|.*.jpeg|.*.png|data/.*.gif|.*.avi|.*.ogg|.*.webm|.*.mp4|.*.mp3|.*.mkv|.*.eot) not allowed, because it has 12 .* repetition patterns. Possible fix: .*(epub|jpg|jpeg|png|data/gif|avi|ogg|webm|mp4|mp3|mkv|eot)
+assert not testSafePattern('(.*.epub|.*.jpg|.*.jpeg|.*.png|data/.*.gif|.*.avi|.*.ogg|.*.webm|.*.mp4|.*.mp3|.*.mkv|.*.eot)')
+assert testSafePattern('.*(epub|jpg|jpeg|png|data/gif|avi|ogg|webm|mp4|mp3|mkv|eot)')
+
+# https://github.com/HelloZeroNet/ZeroNet/issues/2757
+assert not testSafePattern('a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+assert not testSafePattern('a?a?a?a?a?a?a?x.{0,1}x.{0,1}x.{0,1}')
+assert testSafePattern('a?a?a?a?a?a?a?x.{0,1}x.{0,1}')
+assert not testSafePattern('a?a?a?a?a?a?a?x.*x.*x.*')
+assert testSafePattern('a?a?a?a?a?a?a?x.*x.*')
+
+################################################################################
diff --git a/src/util/SelectiveLogger.py b/src/util/SelectiveLogger.py
new file mode 100644
index 00000000..fcdcba0a
--- /dev/null
+++ b/src/util/SelectiveLogger.py
@@ -0,0 +1,43 @@
+import logging
+import re
+
+log_level_raising_rules = []
+
+def addLogLevelRaisingRule(rule, level=None):
+ if level is None:
+ level = logging.INFO
+ log_level_raising_rules.append({
+ "rule": rule,
+ "level": level
+ })
+
+def matchLogLevelRaisingRule(name):
+ for rule in log_level_raising_rules:
+ if isinstance(rule["rule"], re.Pattern):
+ if rule["rule"].search(name):
+ return rule["level"]
+ else:
+ if rule["rule"] == name:
+ return rule["level"]
+ return None
+
+class SelectiveLogger(logging.getLoggerClass()):
+ def __init__(self, name, level=logging.NOTSET):
+ return super().__init__(name, level)
+
+ def raiseLevel(self, level):
+ raised_level = matchLogLevelRaisingRule(self.name)
+ if raised_level is not None:
+ if level < raised_level:
+ level = raised_level
+ return level
+
+ def isEnabledFor(self, level):
+ level = self.raiseLevel(level)
+ return super().isEnabledFor(level)
+
+ def _log(self, level, msg, args, **kwargs):
+ level = self.raiseLevel(level)
+ return super()._log(level, msg, args, **kwargs)
+
+logging.setLoggerClass(SelectiveLogger)
diff --git a/src/util/__init__.py b/src/util/__init__.py
index ab8a8b88..f00c1459 100644
--- a/src/util/__init__.py
+++ b/src/util/__init__.py
@@ -1,4 +1,5 @@
from .Cached import Cached
+from .CircularIterator import CircularIterator
from .Event import Event
from .Noparallel import Noparallel
from .Pooled import Pooled
diff --git a/src/util/helper.py b/src/util/helper.py
index 61455b08..f44bcfce 100644
--- a/src/util/helper.py
+++ b/src/util/helper.py
@@ -290,7 +290,8 @@ local_ip_pattern = re.compile(r"^127\.|192\.168\.|10\.|172\.1[6-9]\.|172\.2[0-9]
def isPrivateIp(ip):
return local_ip_pattern.match(ip)
-
+# XXX: Deprecated. Use ConnectionServer.getIpType() instead.
+# To be removed in 0.9.0
def getIpType(ip):
if ip.endswith(".onion"):
return "onion"
diff --git a/trackers.txt b/trackers.txt
deleted file mode 100644
index a42f8ca4..00000000
--- a/trackers.txt
+++ /dev/null
@@ -1,142 +0,0 @@
-udp://tracker.opentrackr.org:1337/announce
-udp://explodie.org:6969/announce
-udp://open.stealth.si:80/announce
-http://tracker.ipv6tracker.ru:80/announce
-udp://tracker.birkenwald.de:6969/announce
-udp://tracker.moeking.me:6969/announce
-http://tracker.bt4g.com:2095/announce
-https://tracker.nanoha.org:443/announce
-http://tracker.files.fm:6969/announce
-http://open.acgnxtracker.com:80/announce
-udp://tracker.army:6969/announce
-udp://fe.dealclub.de:6969/announce
-udp://tracker.leech.ie:1337/announce
-udp://tracker.altrosky.nl:6969/announce
-https://tracker.cyber-hub.net:443/announce
-https://tracker.lilithraws.cf:443/announce
-http://bt.okmp3.ru:2710/announce
-udp://vibe.sleepyinternetfun.xyz:1738/announce
-udp://open.publictracker.xyz:6969/announce
-udp://tracker.bitsearch.to:1337/announce
-udp://tracker.pomf.se:80/announce
-https://tr.burnabyhighstar.com:443/announce
-https://tr.abiir.top:443/announce
-udp://open.free-tracker.ga:6969/announce
-http://i-p-v-6.tk:6969/announce
-http://open-v6.demonoid.ch:6969/announce
-udp://aarsen.me:6969/announce
-udp://htz3.noho.st:6969/announce
-udp://uploads.gamecoast.net:6969/announce
-udp://mail.zasaonsk.ga:6969/announce
-udp://tracker.joybomb.tw:6969/announce
-udp://tracker.jonaslsa.com:6969/announce
-udp://leefafa.tk:6969/announce
-udp://carr.codes:6969/announce
-https://tr.fuckbitcoin.xyz:443/announce
-udp://tracker.cubonegro.xyz:6969/announce
-udp://tracker.skynetcloud.site:6969/announce
-http://tracker4.itzmx.com:2710/announce
-https://tracker.lilithraws.org:443/announce
-udp://tracker.novaopcj.eu.org:6969/announce
-udp://exodus.desync.com:6969/announce
-http://t.acg.rip:6699/announce
-udp://tracker2.dler.com:80/announce
-udp://6ahddutb1ucc3cp.ru:6969/announce
-udp://tracker.blacksparrowmedia.net:6969/announce
-http://fxtt.ru:80/announce
-udp://tracker.auctor.tv:6969/announce
-udp://torrentclub.space:6969/announce
-udp://zecircle.xyz:6969/announce
-udp://psyco.fr:6969/announce
-udp://fh2.cmp-gaming.com:6969/announce
-udp://new-line.net:6969/announce
-udp://torrents.artixlinux.org:6969/announce
-udp://bt.ktrackers.com:6666/announce
-udp://static.54.161.216.95.clients.your-server.de:6969/announce
-udp://cpe-104-34-3-152.socal.res.rr.com:6969/announce
-http://t.overflow.biz:6969/announce
-udp://tracker1.myporn.club:9337/announce
-udp://moonburrow.club:6969/announce
-udp://tracker.artixlinux.org:6969/announce
-https://t1.hloli.org:443/announce
-udp://bt1.archive.org:6969/announce
-udp://tracker.theoks.net:6969/announce
-udp://tracker.4.babico.name.tr:3131/announce
-udp://buddyfly.top:6969/announce
-udp://ipv6.tracker.harry.lu:80/announce
-udp://public.publictracker.xyz:6969/announce
-udp://mail.artixlinux.org:6969/announce
-udp://v1046920.hosted-by-vdsina.ru:6969/announce
-udp://tracker.cyberia.is:6969/announce
-udp://tracker.beeimg.com:6969/announce
-udp://creative.7o7.cx:6969/announce
-udp://open.dstud.io:6969/announce
-udp://laze.cc:6969/announce
-udp://download.nerocloud.me:6969/announce
-udp://cutscloud.duckdns.org:6969/announce
-https://tracker.jiesen.life:8443/announce
-udp://jutone.com:6969/announce
-udp://wepzone.net:6969/announce
-udp://ipv4.tracker.harry.lu:80/announce
-udp://tracker.tcp.exchange:6969/announce
-udp://f1sh.de:6969/announce
-udp://movies.zsw.ca:6969/announce
-https://tracker1.ctix.cn:443/announce
-udp://sanincode.com:6969/announce
-udp://www.torrent.eu.org:451/announce
-udp://open.4ever.tk:6969/announce
-https://tracker2.ctix.cn:443/announce
-udp://bt2.archive.org:6969/announce
-http://t.nyaatracker.com:80/announce
-udp://yahor.ftp.sh:6969/announce
-udp://tracker.openbtba.com:6969/announce
-udp://tracker.dler.com:6969/announce
-udp://tracker-udp.gbitt.info:80/announce
-udp://tracker.srv00.com:6969/announce
-udp://tracker.pimpmyworld.to:6969/announce
-http://tracker.gbitt.info:80/announce
-udp://tracker6.lelux.fi:6969/announce
-http://tracker.vrpnet.org:6969/announce
-http://00.xxtor.com:443/announce
-http://vps02.net.orel.ru:80/announce
-udp://tracker.yangxiaoguozi.cn:6969/announce
-udp://rep-art.ynh.fr:6969/announce
-https://tracker.imgoingto.icu:443/announce
-udp://mirror.aptus.co.tz:6969/announce
-udp://tracker.lelux.fi:6969/announce
-udp://tracker.torrent.eu.org:451/announce
-udp://admin.52ywp.com:6969/announce
-udp://thouvenin.cloud:6969/announce
-http://vps-dd0a0715.vps.ovh.net:6969/announce
-udp://bubu.mapfactor.com:6969/announce
-udp://94-227-232-84.access.telenet.be:6969/announce
-udp://epider.me:6969/announce
-udp://camera.lei001.com:6969/announce
-udp://tamas3.ynh.fr:6969/announce
-https://tracker.tamersunion.org:443/announce
-udp://ftp.pet:2710/announce
-udp://p4p.arenabg.com:1337/announce
-http://tracker.mywaifu.best:6969/announce
-udp://tracker.monitorit4.me:6969/announce
-udp://ipv6.tracker.monitorit4.me:6969/announce
-zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441
-zero://2kcb2fqesyaevc4lntogupa4mkdssth2ypfwczd2ov5a3zo6ytwwbayd.onion:15441
-zero://5vczpwawviukvd7grfhsfxp7a6huz77hlis4fstjkym5kmf4pu7i7myd.onion:15441
-zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441
-zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441
-zero://tl74auz4tyqv4bieeclmyoe4uwtoc2dj7fdqv4nc4gl5j2bwg2r26bqd.onion:15441
-zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441
-zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441
-zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441
-zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441
-zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441
-zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445
-zero://hb6ozikfiaafeuqvgseiik4r46szbpjfu66l67wjinnyv6dtopuwhtqd.onion:15445
-zero://qn65si4gtcwdiliq7vzrwu62qrweoxb6tx2cchwslaervj6szuje66qd.onion:26117
-zero://s3j2s5pjdfesbsmaqx6alsumaxxdxibmhv4eukmqpv3vqj6f627qx5yd.onion:15441
-zero://agufghdtniyfwty3wk55drxxwj2zxgzzo7dbrtje73gmvcpxy4ngs4ad.onion:15441
-zero://kgsvasoakvj4gnjiy7zemu34l3hq46dn5eauqkn76jpowmilci5t2vqd.onion:15445
-zero://dslesoe72bdfwfu4cfqa2wpd4hr3fhlu4zv6mfsjju5xlpmssouv36qd.onion:15441
-zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441
-zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441
-zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441
\ No newline at end of file
diff --git a/zeronet.py b/zeronet.py
index 457efb19..dacd2096 100755
--- a/zeronet.py
+++ b/zeronet.py
@@ -66,7 +66,7 @@ def displayErrorMessage(err, error_log_path):
res = ctypes.windll.user32.MessageBoxW(0, err_title, "ZeroNet error", MB_YESNOCANCEL | MB_ICONEXCLAIMATION)
if res == ID_YES:
import webbrowser
- report_url = "https://github.com/ZeroNetX/ZeroNet/issues/new?assignees=&labels=&template=bug-report.md&title=%s"
+ report_url = "https://github.com/HelloZeroNet/ZeroNet/issues/new?assignees=&labels=&template=bug-report.md&title=%s"
webbrowser.open(report_url % urllib.parse.quote("Unhandled exception: %s" % err_message))
if res in [ID_YES, ID_NO]:
subprocess.Popen(['notepad.exe', error_log_path])