diff --git a/.forgejo/workflows/build-on-commit.yml b/.forgejo/workflows/build-on-commit.yml new file mode 100644 index 00000000..e8f0d2e3 --- /dev/null +++ b/.forgejo/workflows/build-on-commit.yml @@ -0,0 +1,40 @@ +name: Build Docker Image on Commit + +on: + push: + branches: + - main + tags: + - '!' # Exclude tags + +jobs: + build-and-publish: + runs-on: docker-builder + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set REPO_VARS + id: repo-url + run: | + echo "REPO_HOST=$(echo "${{ github.server_url }}" | sed 's~http[s]*://~~g')" >> $GITHUB_ENV + echo "REPO_PATH=${{ github.repository }}" >> $GITHUB_ENV + + - name: Login to OCI registry + run: | + echo "${{ secrets.OCI_TOKEN }}" | docker login $REPO_HOST -u "${{ secrets.OCI_USER }}" --password-stdin + + - name: Build and push Docker images + run: | + # Build Docker image with commit SHA + docker build -t $REPO_HOST/$REPO_PATH:${{ github.sha }} . + docker push $REPO_HOST/$REPO_PATH:${{ github.sha }} + + # Build Docker image with nightly tag + docker tag $REPO_HOST/$REPO_PATH:${{ github.sha }} $REPO_HOST/$REPO_PATH:nightly + docker push $REPO_HOST/$REPO_PATH:nightly + + # Remove local images to save storage + docker rmi $REPO_HOST/$REPO_PATH:${{ github.sha }} + docker rmi $REPO_HOST/$REPO_PATH:nightly diff --git a/.forgejo/workflows/build-on-tag.yml b/.forgejo/workflows/build-on-tag.yml new file mode 100644 index 00000000..888102b6 --- /dev/null +++ b/.forgejo/workflows/build-on-tag.yml @@ -0,0 +1,37 @@ +name: Build and Publish Docker Image on Tag + +on: + push: + tags: + - '*' + +jobs: + build-and-publish: + runs-on: docker-builder + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set REPO_VARS + id: repo-url + run: | + echo "REPO_HOST=$(echo "${{ github.server_url }}" | sed 's~http[s]*://~~g')" >> $GITHUB_ENV + echo "REPO_PATH=${{ github.repository }}" >> $GITHUB_ENV + + - name: Login to OCI registry + run: | + echo "${{ secrets.OCI_TOKEN }}" | docker login $REPO_HOST -u "${{ secrets.OCI_USER }}" --password-stdin + + - name: Build and push Docker image + run: | + TAG=${{ github.ref_name }} # Get the tag name from the context + # Build and push multi-platform Docker images + docker build -t $REPO_HOST/$REPO_PATH:$TAG --push . + # Tag and push latest + docker tag $REPO_HOST/$REPO_PATH:$TAG $REPO_HOST/$REPO_PATH:latest + docker push $REPO_HOST/$REPO_PATH:latest + + # Remove the local image to save storage + docker rmi $REPO_HOST/$REPO_PATH:$TAG + docker rmi $REPO_HOST/$REPO_PATH:latest \ No newline at end of file diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..aab991d5 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,10 @@ +github: canewsin +patreon: # Replace with a single Patreon username e.g., user1 +open_collective: # Replace with a single Open Collective username e.g., user1 +ko_fi: canewsin +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: canewsin +issuehunt: # Replace with a single IssueHunt username e.g., user1 +otechie: # Replace with a single Otechie username e.g., user1 +custom: ['https://paypal.me/PramUkesh', 'https://zerolink.ml/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/'] diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..b97ad556 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,33 @@ +--- +name: Bug report +about: Create a report to help us improve ZeroNet +title: '' +labels: '' +assignees: '' + +--- + +### Step 1: Please describe your environment + + * ZeroNet version: _____ + * Operating system: _____ + * Web browser: _____ + * Tor status: not available/always/disabled + * Opened port: yes/no + * Special configuration: ____ + +### Step 2: Describe the problem: + +#### Steps to reproduce: + + 1. _____ + 2. _____ + 3. _____ + +#### Observed Results: + + * What happened? This could be a screenshot, a description, log output (you can send log/debug.log file to hello@zeronet.io if necessary), etc. + +#### Expected Results: + + * What did you expect to happen? diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..fe7c8178 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for ZeroNet +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..27b5c924 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ py3-latest ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ py3-latest ] + schedule: + - cron: '32 19 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..2bdcaf95 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,51 @@ +name: tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-20.04 + strategy: + max-parallel: 16 + matrix: + python-version: ["3.7", "3.8", "3.9"] + + steps: + - name: Checkout ZeroNet + uses: actions/checkout@v2 + with: + submodules: "true" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Prepare for installation + run: | + python3 -m pip install setuptools + python3 -m pip install --upgrade pip wheel + python3 -m pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium + + - name: Install + run: | + python3 -m pip install --upgrade -r requirements.txt + python3 -m pip list + + - name: Prepare for tests + run: | + openssl version -a + echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6 + + - name: Test + run: | + catchsegv python3 -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini + export ZERONET_LOG_DIR="log/CryptMessage"; catchsegv python3 -m pytest -x plugins/CryptMessage/Test + export ZERONET_LOG_DIR="log/Bigfile"; catchsegv python3 -m pytest -x plugins/Bigfile/Test + export ZERONET_LOG_DIR="log/AnnounceLocal"; catchsegv python3 -m pytest -x plugins/AnnounceLocal/Test + export ZERONET_LOG_DIR="log/OptionalManager"; catchsegv python3 -m pytest -x plugins/OptionalManager/Test + export ZERONET_LOG_DIR="log/Multiuser"; mv plugins/disabled-Multiuser plugins/Multiuser && catchsegv python -m pytest -x plugins/Multiuser/Test + export ZERONET_LOG_DIR="log/Bootstrapper"; mv plugins/disabled-Bootstrapper plugins/Bootstrapper && catchsegv python -m pytest -x plugins/Bootstrapper/Test + find src -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')" + find plugins -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')" + flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..636cd115 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# Log files +**/*.log + +# Hidden files +.* +!/.forgejo +!/.github +!/.gitignore +!/.travis.yml +!/.gitlab-ci.yml + +# Temporary files +*.bak + +# Data dir +data/* +*.db + +# Virtualenv +env/* + +# Tor data +tools/tor/data + +# PhantomJS, downloaded manually for unit tests +tools/phantomjs + +# ZeroNet config file +zeronet.conf + +# ZeroNet log files +log/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..f3e1ed29 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,48 @@ +stages: + - test + +.test_template: &test_template + stage: test + before_script: + - pip install --upgrade pip wheel + # Selenium and requests can't be installed without a requests hint on Python 3.4 + - pip install --upgrade requests>=2.22.0 + - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium + - pip install --upgrade -r requirements.txt + script: + - pip list + - openssl version -a + - python -m pytest -x plugins/CryptMessage/Test --color=yes + - python -m pytest -x plugins/Bigfile/Test --color=yes + - python -m pytest -x plugins/AnnounceLocal/Test --color=yes + - python -m pytest -x plugins/OptionalManager/Test --color=yes + - python -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini --color=yes + - mv plugins/disabled-Multiuser plugins/Multiuser + - python -m pytest -x plugins/Multiuser/Test --color=yes + - mv plugins/disabled-Bootstrapper plugins/Bootstrapper + - python -m pytest -x plugins/Bootstrapper/Test --color=yes + - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/ + +test:py3.4: + image: python:3.4.3 + <<: *test_template + +test:py3.5: + image: python:3.5.7 + <<: *test_template + +test:py3.6: + image: python:3.6.9 + <<: *test_template + +test:py3.7-openssl1.1.0: + image: python:3.7.0b5 + <<: *test_template + +test:py3.7-openssl1.1.1: + image: python:3.7.4 + <<: *test_template + +test:py3.8: + image: python:3.8.0b3 + <<: *test_template \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..2c602a5a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "plugins"] + path = plugins + url = https://github.com/ZeroNetX/ZeroNet-Plugins.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..bdaafa22 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,47 @@ +language: python +python: + - 3.4 + - 3.5 + - 3.6 + - 3.7 + - 3.8 +services: + - docker +cache: pip +before_install: + - pip install --upgrade pip wheel + - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium + # - docker build -t zeronet . + # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet +install: + - pip install --upgrade -r requirements.txt + - pip list +before_script: + - openssl version -a + # Add an IPv6 config - see the corresponding Travis issue + # https://github.com/travis-ci/travis-ci/issues/8361 + - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then + sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; + fi +script: + - catchsegv python -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini + - export ZERONET_LOG_DIR="log/CryptMessage"; catchsegv python -m pytest -x plugins/CryptMessage/Test + - export ZERONET_LOG_DIR="log/Bigfile"; catchsegv python -m pytest -x plugins/Bigfile/Test + - export ZERONET_LOG_DIR="log/AnnounceLocal"; catchsegv python -m pytest -x plugins/AnnounceLocal/Test + - export ZERONET_LOG_DIR="log/OptionalManager"; catchsegv python -m pytest -x plugins/OptionalManager/Test + - export ZERONET_LOG_DIR="log/Multiuser"; mv plugins/disabled-Multiuser plugins/Multiuser && catchsegv python -m pytest -x plugins/Multiuser/Test + - export ZERONET_LOG_DIR="log/Bootstrapper"; mv plugins/disabled-Bootstrapper plugins/Bootstrapper && catchsegv python -m pytest -x plugins/Bootstrapper/Test + - find src -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')" + - find plugins -name "*.json" | xargs -n 1 python3 -c "import json, sys; print(sys.argv[1], end=' '); json.load(open(sys.argv[1])); print('[OK]')" + - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude=src/lib/pyaes/ +after_failure: + - zip -r log.zip log/ + - curl --upload-file ./log.zip https://transfer.sh/log.zip +after_success: + - codecov + - coveralls --rcfile=src/Test/coverage.ini +notifications: + email: + recipients: + hello@zeronet.io + on_success: change diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..6974d18a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,649 @@ +### ZeroNet 0.9.0 (2023-07-12) Rev4630 + - Fix RDos Issue in Plugins https://github.com/ZeroNetX/ZeroNet-Plugins/pull/9 + - Add trackers to Config.py for failsafety incase missing trackers.txt + - Added Proxy links + - Fix pysha3 dep installation issue + - FileRequest -> Remove Unnecessary check, Fix error wording + - Fix Response when site is missing for `actionAs` + + +### ZeroNet 0.8.5 (2023-02-12) Rev4625 + - Fix(https://github.com/ZeroNetX/ZeroNet/pull/202) for SSL cert gen failed on Windows. + - default theme-class for missing value in `users.json`. + - Fetch Stats Plugin changes. + +### ZeroNet 0.8.4 (2022-12-12) Rev4620 + - Increase Minimum Site size to 25MB. + +### ZeroNet 0.8.3 (2022-12-11) Rev4611 + - main.py -> Fix accessing unassigned varible + - ContentManager -> Support for multiSig + - SiteStrorage.py -> Fix accessing unassigned varible + - ContentManager.py Improve Logging of Valid Signers + +### ZeroNet 0.8.2 (2022-11-01) Rev4610 + - Fix Startup Error when plugins dir missing + - Move trackers to seperate file & Add more trackers + - Config:: Skip loading missing tracker files + - Added documentation for getRandomPort fn + +### ZeroNet 0.8.1 (2022-10-01) Rev4600 + - fix readdress loop (cherry-pick previously added commit from conservancy) + - Remove Patreon badge + - Update README-ru.md (#177) + - Include inner_path of failed request for signing in error msg and response + - Don't Fail Silently When Cert is Not Selected + - Console Log Updates, Specify min supported ZeroNet version for Rust version Protocol Compatibility + - Update FUNDING.yml + +### ZeroNet 0.8.0 (2022-05-27) Rev4591 + - Revert File Open to catch File Access Errors. + +### ZeroNet 0.7.9-patch (2022-05-26) Rev4586 + - Use xescape(s) from zeronet-conservancy + - actionUpdate response Optimisation + - Fetch Plugins Repo Updates + - Fix Unhandled File Access Errors + - Create codeql-analysis.yml + +### ZeroNet 0.7.9 (2022-05-26) Rev4585 + - Rust Version Compatibility for update Protocol msg + - Removed Non Working Trakers. + - Dynamically Load Trackers from Dashboard Site. + - Tracker Supply Improvements. + - Fix Repo Url for Bug Report + - First Party Tracker Update Service using Dashboard Site. + - remove old v2 onion service [#158](https://github.com/ZeroNetX/ZeroNet/pull/158) + +### ZeroNet 0.7.8 (2022-03-02) Rev4580 + - Update Plugins with some bug fixes and Improvements + +### ZeroNet 0.7.6 (2022-01-12) Rev4565 + - Sync Plugin Updates + - Clean up tor v3 patch [#115](https://github.com/ZeroNetX/ZeroNet/pull/115) + - Add More Default Plugins to Repo + - Doubled Site Publish Limits + - Update ZeroNet Repo Urls [#103](https://github.com/ZeroNetX/ZeroNet/pull/103) + - UI/UX: Increases Size of Notifications Close Button [#106](https://github.com/ZeroNetX/ZeroNet/pull/106) + - Moved Plugins to Seperate Repo + - Added `access_key` variable in Config, this used to access restrited plugins when multiuser plugin is enabled. When MultiUserPlugin is enabled we cannot access some pages like /Stats, this key will remove such restriction with access key. + - Added `last_connection_id_current_version` to ConnectionServer, helpful to estimate no of connection from current client version. + - Added current version: connections to /Stats page. see the previous point. + +### ZeroNet 0.7.5 (2021-11-28) Rev4560 + - Add more default trackers + - Change default homepage address to `1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d` + - Change default update site address to `1Update8crprmciJHwp2WXqkx2c4iYp18` + +### ZeroNet 0.7.3 (2021-11-28) Rev4555 + - Fix xrange is undefined error + - Fix Incorrect viewport on mobile while loading + - Tor-V3 Patch by anonymoose + + +### ZeroNet 0.7.1 (2019-07-01) Rev4206 +### Added + - Built-in logging console in the web UI to see what's happening in the background. (pull down top-right 0 button to see it) + - Display database rebuild errors [Thanks to Lola] + - New plugin system that allows to install and manage builtin/third party extensions to the ZeroNet client using the web interface. + - Support multiple trackers_file + - Add OpenSSL 1.1 support to CryptMessage plugin based on Bitmessage modifications [Thanks to radfish] + - Display visual error message on startup errors + - Fix max opened files changing on Windows platform + - Display TLS1.3 compatibility on /Stats page + - Add fake SNI and ALPN to peer connections to make it more like standard https connections + - Hide and ignore tracker_proxy setting in Tor: Always mode as it's going to use Tor anyway. + - Deny websocket connections from unknown origins + - Restrict open_browser values to avoid RCE on sandbox escape + - Offer access web interface by IP address in case of unknown host + - Link to site's sidebar with "#ZeroNet:OpenSidebar" hash + +### Changed + - Allow .. in file names [Thanks to imachug] + - Change unstable trackers + - More clean errors on sites.json/users.json load error + - Various tweaks for tracker rating on unstable connections + - Use OpenSSL 1.1 dlls from default Python Windows distribution if possible + - Re-factor domain resolving for easier domain plugins + - Disable UDP connections if --proxy is used + - New, decorator-based Websocket API permission system to avoid future typo mistakes + +### Fixed + - Fix parsing config lines that have no value + - Fix start.py [Thanks to imachug] + - Allow multiple values of the same key in the config file [Thanks ssdifnskdjfnsdjk for reporting] + - Fix parsing config file lines that has % in the value [Thanks slrslr for reporting] + - Fix bootstrapper plugin hash reloads [Thanks geekless for reporting] + - Fix CryptMessage plugin OpenSSL dll loading on Windows (ZeroMail errors) [Thanks cxgreat2014 for reporting] + - Fix startup error when using OpenSSL 1.1 [Thanks to imachug] + - Fix a bug that did not loaded merged site data for 5 sec after the merged site got added + - Fix typo that allowed to add new plugins in public proxy mode. [Thanks styromaniac for reporting] + - Fix loading non-big files with "|all" postfix [Thanks to krzotr] + - Fix OpenSSL cert generation error crash by change Windows console encoding to utf8 + +#### Wrapper html injection vulnerability [Reported by ivanq] + +In ZeroNet before rev4188 the wrapper template variables was rendered incorrectly. + +Result: The opened site was able to gain WebSocket connection with unrestricted ADMIN/NOSANDBOX access, change configuration values and possible RCE on client's machine. + +Fix: Fixed the template rendering code, disallowed WebSocket connections from unknown locations, restricted open_browser configuration values to avoid possible RCE in case of sandbox escape. + +Note: The fix is also back ported to ZeroNet Py 2.x version (Rev3870) + + +### ZeroNet 0.7.0 (2019-06-12) Rev4106 (First release targeting Python 3.4+) +### Added + - 5-10x faster signature verification by using libsecp256k1 (Thanks to ZeroMux) + - Generated SSL certificate randomization to avoid protocol filters (Thanks to ValdikSS) + - Offline mode + - P2P source code update using ZeroNet protocol + - ecdsaSign/Verify commands to CryptMessage plugin (Thanks to imachug) + - Efficient file rename: change file names instead of re-downloading the file. + - Make redirect optional on site cloning (Thanks to Lola) + - EccPrivToPub / EccPubToPriv functions (Thanks to imachug) + - Detect and change dark/light theme based on OS setting (Thanks to filips123) + +### Changed + - Re-factored code to Python3 runtime (compatible with Python 3.4-3.8) + - More safe database sync mode + - Removed bundled third-party libraries where it's possible + - Use lang=en instead of lang={lang} in urls to avoid url encode problems + - Remove environment details from error page + - Don't push content.json updates larger than 10kb to significantly reduce bw usage for site with many files + +### Fixed + - Fix sending files with \0 characters + - Security fix: Escape error detail to avoid XSS (reported by krzotr) + - Fix signature verification using libsecp256k1 for compressed addresses (mostly certificates generated in the browser) + - Fix newsfeed if you have more than 1000 followed topic/post on one site. + - Fix site download as zip file + - Fix displaying sites with utf8 title + - Error message if dbRebuild fails (Thanks to Lola) + - Fix browser reopen if executing start.py again. (Thanks to imachug) + + +### ZeroNet 0.6.5 (2019-02-16) Rev3851 (Last release targeting Python 2.7.x) +### Added + - IPv6 support in peer exchange, bigfiles, optional file finding, tracker sharing, socket listening and connecting (based on tangdou1 modifications) + - New tracker database format with IPv6 support + - Display notification if there is an unpublished modification for your site + - Listen and shut down normally for SIGTERM (Thanks to blurHY) + - Support tilde `~` in filenames (by d14na) + - Support map for Namecoin subdomain names (Thanks to lola) + - Add log level to config page + - Support `{data}` for data dir variable in trackers_file value + - Quick check content.db on startup and rebuild if necessary + - Don't show meek proxy option if the tor client does not supports it + +### Changed + - Refactored port open checking with IPv6 support + - Consider non-local IPs as external even is the open port check fails (for CJDNS and Yggdrasil support) + - Add IPv6 tracker and change unstable tracker + - Don't correct sent local time with the calculated time correction + - Disable CSP for Edge + - Only support CREATE commands in dbschema indexes node and SELECT from storage.query + +### Fixed + - Check the length of master seed when executing cryptGetPrivatekey CLI command + - Only reload source code on file modification / creation + - Detection and issue warning for latest no-script plugin + - Fix atomic write of a non-existent file + - Fix sql queries with lots of variables and sites with lots of content.json + - Fix multi-line parsing of zeronet.conf + - Fix site deletion from users.json + - Fix site cloning before site downloaded (Reported by unsystemizer) + - Fix queryJson for non-list nodes (Reported by MingchenZhang) + + +## ZeroNet 0.6.4 (2018-10-20) Rev3660 +### Added + - New plugin: UiConfig. A web interface that allows changing ZeroNet settings. + - New plugin: AnnounceShare. Share trackers between users, automatically announce client's ip as tracker if Bootstrapper plugin is enabled. + - Global tracker stats on ZeroHello: Include statistics from all served sites instead of displaying request statistics only for one site. + - Support custom proxy for trackers. (Configurable with /Config) + - Adding peers to sites manually using zeronet_peers get parameter + - Copy site address with peers link on the sidebar. + - Zip file listing and streaming support for Bigfiles. + - Tracker statistics on /Stats page + - Peer reputation save/restore to speed up sync time after startup. + - Full support fileGet, fileList, dirList calls on tar.gz/zip files. + - Archived_before support to user content rules to allow deletion of all user files before the specified date + - Show and manage "Connecting" sites on ZeroHello + - Add theme support to ZeroNet sites + - Dark theme for ZeroHello, ZeroBlog, ZeroTalk + +### Changed + - Dynamic big file allocation: More efficient storage usage by don't pre-allocate the whole file at the beginning, but expand the size as the content downloads. + - Reduce the request frequency to unreliable trackers. + - Only allow 5 concurrent checkSites to run in parallel to reduce load under Tor/slow connection. + - Stop site downloading if it reached 95% of site limit to avoid download loop for sites out of limit + - The pinned optional files won't be removed from download queue after 30 retries and won't be deleted even if the site owner removes it. + - Don't remove incomplete (downloading) sites on startup + - Remove --pin_bigfile argument as big files are automatically excluded from optional files limit. + +### Fixed + - Trayicon compatibility with latest gevent + - Request number counting for zero:// trackers + - Peer reputation boost for zero:// trackers. + - Blocklist of peers loaded from peerdb (Thanks tangdou1 for report) + - Sidebar map loading on foreign languages (Thx tangdou1 for report) + - FileGet on non-existent files (Thanks mcdev for reporting) + - Peer connecting bug for sites with low amount of peers + +#### "The Vacation" Sandbox escape bug [Reported by GitCenter / Krixano / ZeroLSTN] + +In ZeroNet 0.6.3 Rev3615 and earlier as a result of invalid file type detection, a malicious site could escape the iframe sandbox. + +Result: Browser iframe sandbox escape + +Applied fix: Replaced the previous, file extension based file type identification with a proper one. + +Affected versions: All versions before ZeroNet Rev3616 + + +## ZeroNet 0.6.3 (2018-06-26) +### Added + - New plugin: ContentFilter that allows to have shared site and user block list. + - Support Tor meek proxies to avoid tracker blocking of GFW + - Detect network level tracker blocking and easy setting meek proxy for tracker connections. + - Support downloading 2GB+ sites as .zip (Thx to Radtoo) + - Support ZeroNet as a transparent proxy (Thx to JeremyRand) + - Allow fileQuery as CORS command (Thx to imachug) + - Windows distribution includes Tor and meek client by default + - Download sites as zip link to sidebar + - File server port randomization + - Implicit SSL for all connection + - fileList API command for zip files + - Auto download bigfiles size limit on sidebar + - Local peer number to the sidebar + - Open site directory button in sidebar + +### Changed + - Switched to Azure Tor meek proxy as Amazon one became unavailable + - Refactored/rewritten tracker connection manager + - Improved peer discovery for optional files without opened port + - Also delete Bigfile's piecemap on deletion + +### Fixed + - Important security issue: Iframe sandbox escape [Reported by Ivanq / gitcenter] + - Local peer discovery when running multiple clients on the same machine + - Uploading small files with Bigfile plugin + - Ctrl-c shutdown when running CLI commands + - High CPU/IO usage when Multiuser plugin enabled + - Firefox back button + - Peer discovery on older Linux kernels + - Optional file handling when multiple files have the same hash_id (first 4 chars of the hash) + - Msgpack 0.5.5 and 0.5.6 compatibility + +## ZeroNet 0.6.2 (2018-02-18) + +### Added + - New plugin: AnnounceLocal to make ZeroNet work without an internet connection on the local network. + - Allow dbQuey and userGetSettings using the `as` API command on different sites with Cors permission + - New config option: `--log_level` to reduce log verbosity and IO load + - Prefer to connect to recent peers from trackers first + - Mark peers with port 1 is also unconnectable for future fix for trackers that do not support port 0 announce + +### Changed + - Don't keep connection for sites that have not been modified in the last week + - Change unreliable trackers to new ones + - Send maximum 10 findhash request in one find optional files round (15sec) + - Change "Unique to site" to "No certificate" for default option in cert selection dialog. + - Dont print warnings if not in debug mode + - Generalized tracker logging format + - Only recover sites from sites.json if they had peers + - Message from local peers does not means internet connection + - Removed `--debug_gevent` and turned on Gevent block logging by default + +### Fixed + - Limit connections to 512 to avoid reaching 1024 limit on windows + - Exception when logging foreign operating system socket errors + - Don't send private (local) IPs on pex + - Don't connect to private IPs in tor always mode + - Properly recover data from msgpack unpacker on file stream start + - Symlinked data directory deletion when deleting site using Windows + - De-duplicate peers before publishing + - Bigfile info for non-existing files + + +## ZeroNet 0.6.1 (2018-01-25) + +### Added + - New plugin: Chart + - Collect and display charts about your contribution to ZeroNet network + - Allow list as argument replacement in sql queries. (Thanks to imachug) + - Newsfeed query time statistics (Click on "From XX sites in X.Xs on ZeroHello) + - New UiWebsocket API command: As to run commands as other site + - Ranged ajax queries for big files + - Filter feed by type and site address + - FileNeed, Bigfile upload command compatibility with merger sites + - Send event on port open / tor status change + - More description on permission request + +### Changed + - Reduce memory usage of sidebar geoip database cache + - Change unreliable tracker to new one + - Don't display Cors permission ask if it already granted + - Avoid UI blocking when rebuilding a merger site + - Skip listing ignored directories on signing + - In Multiuser mode show the seed welcome message when adding new certificate instead of first visit + - Faster async port opening on multiple network interfaces + - Allow javascript modals + - Only zoom sidebar globe if mouse button is pressed down + +### Fixed + - Open port checking error reporting (Thanks to imachug) + - Out-of-range big file requests + - Don't output errors happened on gevent greenlets twice + - Newsfeed skip sites with no database + - Newsfeed queries with multiple params + - Newsfeed queries with UNION and UNION ALL + - Fix site clone with sites larger that 10MB + - Unreliable Websocket connection when requesting files from different sites at the same time + + +## ZeroNet 0.6.0 (2017-10-17) + +### Added + - New plugin: Big file support + - Automatic pinning on Big file download + - Enable TCP_NODELAY for supporting sockets + - actionOptionalFileList API command arguments to list non-downloaded files or only big files + - serverShowdirectory API command arguments to allow to display site's directory in OS file browser + - fileNeed API command to initialize optional file downloading + - wrapperGetAjaxKey API command to request nonce for AJAX request + - Json.gz support for database files + - P2P port checking (Thanks for grez911) + - `--download_optional auto` argument to enable automatic optional file downloading for newly added site + - Statistics for big files and protocol command requests on /Stats + - Allow to set user limitation based on auth_address + +### Changed + - More aggressive and frequent connection timeout checking + - Use out of msgpack context file streaming for files larger than 512KB + - Allow optional files workers over the worker limit + - Automatic redirection to wrapper on nonce_error + - Send websocket event on optional file deletion + - Optimize sites.json saving + - Enable faster C-based msgpack packer by default + - Major optimization on Bootstrapper plugin SQL queries + - Don't reset bad file counter on restart, to allow easier give up on unreachable files + - Incoming connection limit changed from 1000 to 500 to avoid reaching socket limit on Windows + - Changed tracker boot.zeronet.io domain, because zeronet.io got banned in some countries + +#### Fixed + - Sub-directories in user directories + +## ZeroNet 0.5.7 (2017-07-19) +### Added + - New plugin: CORS to request read permission to other site's content + - New API command: userSetSettings/userGetSettings to store site's settings in users.json + - Avoid file download if the file size does not match with the requested one + - JavaScript and wrapper less file access using /raw/ prefix ([Example](http://127.0.0.1:43110/raw/1AsRLpuRxr3pb9p3TKoMXPSWHzh6i7fMGi/en.tar.gz/index.html)) + - --silent command line option to disable logging to stdout + + +### Changed + - Better error reporting on sign/verification errors + - More test for sign and verification process + - Update to OpenSSL v1.0.2l + - Limit compressed files to 6MB to avoid zip/tar.gz bomb + - Allow space, [], () characters in filenames + - Disable cross-site resource loading to improve privacy. [Reported by Beardog108] + - Download directly accessed Pdf/Svg/Swf files instead of displaying them to avoid wrapper escape using in JS in SVG file. [Reported by Beardog108] + - Disallow potentially unsafe regular expressions to avoid ReDoS [Reported by MuxZeroNet] + +### Fixed + - Detecting data directory when running Windows distribution exe [Reported by Plasmmer] + - OpenSSL loading under Android 6+ + - Error on exiting when no connection server started + + +## ZeroNet 0.5.6 (2017-06-15) +### Added + - Callback for certSelect API command + - More compact list formatting in json + +### Changed + - Remove obsolete auth_key_sha512 and signature format + - Improved Spanish translation (Thanks to Pupiloho) + +### Fixed + - Opened port checking (Thanks l5h5t7 & saber28 for reporting) + - Standalone update.py argument parsing (Thanks Zalex for reporting) + - uPnP crash on startup (Thanks Vertux for reporting) + - CoffeeScript 1.12.6 compatibility (Thanks kavamaken & imachug) + - Multi value argument parsing + - Database error when running from directory that contains special characters (Thanks Pupiloho for reporting) + - Site lock violation logging + + +#### Proxy bypass during source upgrade [Reported by ZeroMux] + +In ZeroNet before 0.5.6 during the client's built-in source code upgrade mechanism, +ZeroNet did not respect Tor and/or proxy settings. + +Result: ZeroNet downloaded the update without using the Tor network and potentially leaked the connections. + +Fix: Removed the problematic code line from the updater that removed the proxy settings from the socket library. + +Affected versions: ZeroNet 0.5.5 and earlier, Fixed in: ZeroNet 0.5.6 + + +#### XSS vulnerability using DNS rebinding. [Reported by Beardog108] + +In ZeroNet before 0.5.6 the web interface did not validate the request's Host parameter. + +Result: An attacker using a specially crafted DNS entry could have bypassed the browser's cross-site-scripting protection +and potentially gained access to user's private data stored on site. + +Fix: By default ZeroNet only accept connections from 127.0.0.1 and localhost hosts. +If you bind the ui server to an external interface, then it also adds the first http request's host to the allowed host list +or you can define it manually using --ui_host. + +Affected versions: ZeroNet 0.5.5 and earlier, Fixed in: ZeroNet 0.5.6 + + +## ZeroNet 0.5.5 (2017-05-18) +### Added +- Outgoing socket binding by --bind parameter +- Database rebuilding progress bar +- Protect low traffic site's peers from cleanup closing +- Local site blacklisting +- Cloned site source code upgrade from parent +- Input placeholder support for displayPrompt +- Alternative interaction for wrapperConfirm + +### Changed +- New file priorities for faster site display on first visit +- Don't add ? to url if push/replaceState url starts with # + +### Fixed +- PermissionAdd/Remove admin command requirement +- Multi-line confirmation dialog + + +## ZeroNet 0.5.4 (2017-04-14) +### Added +- Major speed and CPU usage enhancements in Tor always mode +- Send skipped modifications to outdated clients + +### Changed +- Upgrade libs to latest version +- Faster port opening and closing +- Deny site limit modification in MultiUser mode + +### Fixed +- Filling database from optional files +- OpenSSL detection on systems with OpenSSL 1.1 +- Users.json corruption on systems with slow hdd +- Fix leaking files in data directory by webui + + +## ZeroNet 0.5.3 (2017-02-27) +### Added +- Tar.gz/zip packed site support +- Utf8 filenames in archive files +- Experimental --db_mode secure database mode to prevent data loss on systems with unreliable power source. +- Admin user support in MultiUser mode +- Optional deny adding new sites in MultiUser mode + +### Changed +- Faster update and publish times by new socket sharing algorithm + +### Fixed +- Fix missing json_row errors when using Mute plugin + + +## ZeroNet 0.5.2 (2017-02-09) +### Added +- User muting +- Win/Mac signed exe/.app +- Signed commits + +### Changed +- Faster site updates after startup +- New macOS package for 10.10 compatibility + +### Fixed +- Fix "New version just released" popup on page first visit +- Fix disappearing optional files bug (Thanks l5h5t7 for reporting) +- Fix skipped updates on unreliable connections (Thanks P2P for reporting) +- Sandbox escape security fix (Thanks Firebox for reporting) +- Fix error reporting on async websocket functions + + +## ZeroNet 0.5.1 (2016-11-18) +### Added +- Multi language interface +- New plugin: Translation helper for site html and js files +- Per-site favicon + +### Fixed +- Parallel optional file downloading + + +## ZeroNet 0.5.0 (2016-11-08) +### Added +- New Plugin: Allow list/delete/pin/manage files on ZeroHello +- New API commands to follow user's optional files, and query stats for optional files +- Set total size limit on optional files. +- New Plugin: Save peers to database and keep them between restarts to allow more faster optional file search and make it work without trackers +- Rewritten uPnP port opener + close port on exit (Thanks to sirMackk!) +- Lower memory usage by lazy PeerHashfield creation +- Loaded json files statistics and database info at /Stats page + +### Changed +- Separate lock file for better Windows compatibility +- When executing start.py open browser even if ZeroNet is already running +- Keep plugin order after reload to allow plugins to extends an another plug-in +- Only save sites.json if fully loaded to avoid data loss +- Change aletorrenty tracker to a more reliable one +- Much lower findhashid CPU usage +- Pooled downloading of large amount of optional files +- Lots of other optional file changes to make it better +- If we have 1000 peers for a site make cleanup more aggressive +- Use warning instead of error on verification errors +- Push updates to newer clients first +- Bad file reset improvements + +### Fixed +- Fix site deletion errors on startup +- Delay websocket messages until it's connected +- Fix database import if data file contains extra data +- Fix big site download +- Fix diff sending bug (been chasing it for a long time) +- Fix random publish errors when json file contained [] characters +- Fix site delete and siteCreate bug +- Fix file write confirmation dialog + + +## ZeroNet 0.4.1 (2016-09-05) +### Added +- Major core changes to allow fast startup and lower memory usage +- Try to reconnect to Tor on lost connection +- Sidebar fade-in +- Try to avoid incomplete data files overwrite +- Faster database open +- Display user file sizes in sidebar +- Concurrent worker number depends on --connection_limit + +### Changed +- Close databases after 5 min idle time +- Better site size calculation +- Allow "-" character in domains +- Always try to keep connections for sites +- Remove merger permission from merged sites +- Newsfeed scans only last 3 days to speed up database queries +- Updated ZeroBundle-win to Python 2.7.12 + +### Fixed +- Fix for important security problem, which is allowed anyone to publish new content without valid certificate from ID provider. Thanks Kaffie for pointing it out! +- Fix sidebar error when no certificate provider selected +- Skip invalid files on database rebuilding +- Fix random websocket connection error popups +- Fix new siteCreate command +- Fix site size calculation +- Fix port open checking after computer wake up +- Fix --size_limit parsing from command line + + +## ZeroNet 0.4.0 (2016-08-11) +### Added +- Merger site plugin +- Live source code reloading: Faster core development by allowing me to make changes in ZeroNet source code without restarting it. +- New json table format for merger sites +- Database rebuild from sidebar. +- Allow to store custom data directly in json table: Much simpler and faster SQL queries. +- User file archiving: Allows the site owner to archive inactive user's content into single file. (Reducing initial sync time/cpu/memory usage) +- Also trigger onUpdated/update database on file delete. +- Permission request from ZeroFrame API. +- Allow to store extra data in content.json using fileWrite API command. +- Faster optional files downloading +- Use alternative sources (Gogs, Gitlab) to download updates +- Track provided sites/connection and prefer to keep the ones with more sites to reduce connection number + +### Changed +- Keep at least 5 connection per site +- Changed target connection for sites to 10 from 15 +- ZeroHello search function stability/speed improvements +- Improvements for clients with slower HDD + +### Fixed +- Fix IE11 wrapper nonce errors +- Fix sidebar on mobile devices +- Fix site size calculation +- Fix IE10 compatibility +- Windows XP ZeroBundle compatibility (THX to people of China) + + +## ZeroNet 0.3.7 (2016-05-27) +### Changed +- Patch command to reduce bandwidth usage by transfer only the changed lines +- Other cpu/memory optimizations + + +## ZeroNet 0.3.6 (2016-05-27) +### Added +- New ZeroHello +- Newsfeed function + +### Fixed +- Security fixes + + +## ZeroNet 0.3.5 (2016-02-02) +### Added +- Full Tor support with .onion hidden services +- Bootstrap using ZeroNet protocol + +### Fixed +- Fix Gevent 1.0.2 compatibility + + +## ZeroNet 0.3.4 (2015-12-28) +### Added +- AES, ECIES API function support +- PushState and ReplaceState url manipulation support in API +- Multiuser localstorage diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..f288702d --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3f1d3c18 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM alpine:3.15 + +#Base settings +ENV HOME /root + +COPY requirements.txt /root/requirements.txt + +#Install ZeroNet +RUN apk --update --no-cache --no-progress add python3 python3-dev py3-pip gcc g++ autoconf automake libtool libffi-dev musl-dev make tor openssl \ + && pip3 install -r /root/requirements.txt \ + && apk del python3-dev gcc g++ autoconf automake libtool libffi-dev musl-dev make \ + && echo "ControlPort 9051" >> /etc/tor/torrc \ + && echo "CookieAuthentication 1" >> /etc/tor/torrc + +RUN python3 -V \ + && python3 -m pip list \ + && tor --version \ + && openssl version + +#Add Zeronet source +COPY . /root +VOLUME /root/data + +#Control if Tor proxy is started +ENV ENABLE_TOR true + +WORKDIR /root + +#Set upstart command +CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26117 + +#Expose ports +EXPOSE 43110 26117 diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 new file mode 100644 index 00000000..d27b7620 --- /dev/null +++ b/Dockerfile.arm64v8 @@ -0,0 +1,34 @@ +FROM alpine:3.12 + +#Base settings +ENV HOME /root + +COPY requirements.txt /root/requirements.txt + +#Install ZeroNet +RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ + && pip3 install -r /root/requirements.txt \ + && apk del python3-dev gcc libffi-dev musl-dev make \ + && echo "ControlPort 9051" >> /etc/tor/torrc \ + && echo "CookieAuthentication 1" >> /etc/tor/torrc + +RUN python3 -V \ + && python3 -m pip list \ + && tor --version \ + && openssl version + +#Add Zeronet source +COPY . /root +VOLUME /root/data + +#Control if Tor proxy is started +ENV ENABLE_TOR false + +WORKDIR /root + +#Set upstart command +CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 + +#Expose ports +EXPOSE 43110 26552 + diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..0d17b72d --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + + +Additional Conditions : + +Contributing to this repo + This repo is governed by GPLv3, same is located at the root of the ZeroNet git repo, + unless specified separately all code is governed by that license, contributions to this repo + are divided into two key types, key contributions and non-key contributions, key contributions + are which, directly affects the code performance, quality and features of software, + non key contributions include things like translation datasets, image, graphic or video + contributions that does not affect the main usability of software but improves the existing + usability of certain thing or feature, these also include tests written with code, since their + purpose is to check, whether something is working or not as intended. All the non-key contributions + are governed by [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/), unless specified + above, a contribution is ruled by the type of contribution if there is a conflict between two + contributing parties of repo in any case. diff --git a/README-ru.md b/README-ru.md new file mode 100644 index 00000000..7d557727 --- /dev/null +++ b/README-ru.md @@ -0,0 +1,133 @@ +# ZeroNet [![tests](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml/badge.svg)](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [![Docker Pulls](https://img.shields.io/docker/pulls/canewsin/zeronet)](https://hub.docker.com/r/canewsin/zeronet) + +[įŽ€äŊ“中文](./README-zh-cn.md) +[English](./README.md) + +ДĐĩŅ†ĐĩĐŊŅ‚Ņ€Đ°ĐģиСОваĐŊĐŊŅ‹Đĩ вĐĩĐąŅĐ°ĐšŅ‚Ņ‹, иŅĐŋĐžĐģŅŒĐˇŅƒŅŽŅ‰Đ¸Đĩ ĐēŅ€Đ¸ĐŋŅ‚ĐžĐŗŅ€Đ°Ņ„иŅŽ Bitcoin и ĐŋŅ€ĐžŅ‚ĐžĐēĐžĐģ BitTorrent — https://zeronet.dev ([ЗĐĩŅ€ĐēĐ°ĐģĐž в ZeroNet](http://127.0.0.1:43110/1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX/)). В ĐžŅ‚ĐģиŅ‡Đ¸Đ¸ ĐžŅ‚ Bitcoin, ZeroNet'Ņƒ ĐŊĐĩ Ņ‚Ņ€ĐĩĐąŅƒĐĩŅ‚ŅŅ ĐąĐģĐžĐēŅ‡ĐĩĐšĐŊ Đ´ĐģŅ Ņ€Đ°ĐąĐžŅ‚Ņ‹, ОдĐŊĐ°ĐēĐž ĐžĐŊ иŅĐŋĐžĐģŅŒĐˇŅƒĐĩŅ‚ Ņ‚Ņƒ ĐļĐĩ ĐēŅ€Đ¸ĐŋŅ‚ĐžĐŗŅ€Đ°Ņ„иŅŽ, Ņ‡Ņ‚ОйŅ‹ ОйĐĩŅĐŋĐĩŅ‡Đ¸Ņ‚ŅŒ ŅĐžŅ…Ņ€Đ°ĐŊĐŊĐžŅŅ‚ŅŒ и ĐŋŅ€ĐžĐ˛ĐĩŅ€ĐēŅƒ Đ´Đ°ĐŊĐŊŅ‹Ņ…. + +## ЗаŅ‡ĐĩĐŧ? + +- МŅ‹ вĐĩŅ€Đ¸Đŧ в ĐžŅ‚ĐēŅ€Ņ‹Ņ‚ŅƒŅŽ, ŅĐ˛ĐžĐąĐžĐ´ĐŊŅƒŅŽ, и ĐŊĐĩĐŋОддаŅŽŅ‰ŅƒŅŽŅŅ Ņ†ĐĩĐŊСŅƒŅ€Đĩ ŅĐĩŅ‚ŅŒ и ŅĐ˛ŅĐˇŅŒ. +- НĐĩŅ‚ ĐĩдиĐŊОК Ņ‚ĐžŅ‡Đēи ĐžŅ‚ĐēаСа: ĐĄĐ°ĐšŅ‚ ĐžŅŅ‚Đ°Ņ‘Ņ‚ŅŅ ĐžĐŊĐģĐ°ĐšĐŊ, ĐŋĐžĐēĐ° ĐĩĐŗĐž ОйŅĐģŅƒĐļиваĐĩŅ‚ Ņ…ĐžŅ‚Ņ ĐąŅ‹ 1 ĐŋиŅ€. +- НĐĩŅ‚ СаŅ‚Ņ€Đ°Ņ‚ ĐŊĐ° Ņ…ĐžŅŅ‚иĐŊĐŗ: ĐĄĐ°ĐšŅ‚Ņ‹ ОйŅĐģŅƒĐļиваŅŽŅ‚ŅŅ ĐŋĐžŅĐĩŅ‚иŅ‚ĐĩĐģŅĐŧи. +- НĐĩвОСĐŧĐžĐļĐŊĐž ĐžŅ‚ĐēĐģŅŽŅ‡Đ¸Ņ‚ŅŒ: ОĐŊ ĐŊиĐŗĐ´Đĩ, ĐŋĐžŅ‚ĐžĐŧŅƒ Ņ‡Ņ‚Đž ĐžĐŊ вĐĩСдĐĩ. +- ĐĄĐēĐžŅ€ĐžŅŅ‚ŅŒ и вОСĐŧĐžĐļĐŊĐžŅŅ‚ŅŒ Ņ€Đ°ĐąĐžŅ‚Đ°Ņ‚ŅŒ ĐąĐĩС ИĐŊŅ‚ĐĩŅ€ĐŊĐĩŅ‚Đ°: ВŅ‹ ŅĐŧĐžĐļĐĩŅ‚Đĩ ĐŋĐžĐģŅƒŅ‡Đ¸Ņ‚ŅŒ Đ´ĐžŅŅ‚ŅƒĐŋ Đē ŅĐ°ĐšŅ‚Ņƒ, ĐŋĐžŅ‚ĐžĐŧŅƒ Ņ‡Ņ‚Đž ĐĩĐŗĐž ĐēĐžĐŋиŅ Ņ…Ņ€Đ°ĐŊиŅ‚ŅŅ ĐŊĐ° ваŅˆĐĩĐŧ ĐēĐžĐŧĐŋŅŒŅŽŅ‚ĐĩŅ€Đĩ и Ņƒ ваŅˆĐ¸Ņ… ĐŋиŅ€ĐžĐ˛. + +## ОŅĐžĐąĐĩĐŊĐŊĐžŅŅ‚и + +- ОбĐŊОвĐģĐĩĐŊиĐĩ ŅĐ°ĐšŅ‚Ов в Ņ€ĐĩĐ°ĐģŅŒĐŊĐžĐŧ вŅ€ĐĩĐŧĐĩĐŊи +- ПоддĐĩŅ€ĐļĐēĐ° Đ´ĐžĐŧĐĩĐŊОв `.bit` ([Namecoin](https://www.namecoin.org)) +- ЛĐĩĐŗĐēĐ°Ņ ŅƒŅŅ‚Đ°ĐŊОвĐēĐ°: ĐŋŅ€ĐžŅŅ‚Đž Ņ€Đ°ŅĐŋĐ°ĐēŅƒĐšŅ‚Đĩ и СаĐŋŅƒŅŅ‚иŅ‚Đĩ +- КĐģĐžĐŊиŅ€ĐžĐ˛Đ°ĐŊиĐĩ ŅĐ°ĐšŅ‚Ов "в ОдиĐŊ ĐēĐģиĐē" +- БĐĩŅĐŋĐ°Ņ€ĐžĐģŅŒĐŊĐ°Ņ [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) + авŅ‚ĐžŅ€Đ¸ĐˇĐ°Ņ†Đ¸Ņ: ВаŅˆĐ° ŅƒŅ‡ĐĩŅ‚ĐŊĐ°Ņ СаĐŋиŅŅŒ СаŅ‰Đ¸Ņ‰ĐĩĐŊĐ° Ņ‚ОК ĐļĐĩ ĐēŅ€Đ¸ĐŋŅ‚ĐžĐŗŅ€Đ°Ņ„иĐĩĐš, Ņ‡Ņ‚Đž и ваŅˆ Bitcoin-ĐēĐžŅˆĐĩĐģĐĩĐē +- ВŅŅ‚Ņ€ĐžĐĩĐŊĐŊŅ‹Đš SQL-ŅĐĩŅ€Đ˛ĐĩŅ€ Ņ ŅĐ¸ĐŊŅ…Ņ€ĐžĐŊиСаŅ†Đ¸ĐĩĐš Đ´Đ°ĐŊĐŊŅ‹Ņ… P2P: ПозвоĐģŅĐĩŅ‚ ŅƒĐŋŅ€ĐžŅŅ‚иŅ‚ŅŒ Ņ€Đ°ĐˇŅ€Đ°ĐąĐžŅ‚ĐēŅƒ ŅĐ°ĐšŅ‚Đ° и ŅƒŅĐēĐžŅ€Đ¸Ņ‚ŅŒ СаĐŗŅ€ŅƒĐˇĐēŅƒ ŅŅ‚Ņ€Đ°ĐŊиŅ†Ņ‹ +- АĐŊĐžĐŊиĐŧĐŊĐžŅŅ‚ŅŒ: ПоĐģĐŊĐ°Ņ ĐŋОддĐĩŅ€ĐļĐēĐ° ŅĐĩŅ‚и Tor, иŅĐŋĐžĐģŅŒĐˇŅƒŅ ŅĐēŅ€Ņ‹Ņ‚Ņ‹Đĩ ŅĐģŅƒĐļĐąŅ‹ `.onion` вĐŧĐĩŅŅ‚Đž Đ°Đ´Ņ€ĐĩŅĐžĐ˛ IPv4 +- ЗаŅˆĐ¸Ņ„Ņ€ĐžĐ˛Đ°ĐŊĐŊĐžĐĩ TLS ĐŋОдĐēĐģŅŽŅ‡ĐĩĐŊиĐĩ +- АвŅ‚ĐžĐŧĐ°Ņ‚иŅ‡ĐĩŅĐēĐžĐĩ ĐžŅ‚ĐēŅ€Ņ‹Ņ‚иĐĩ UPnP–ĐŋĐžŅ€Ņ‚Đ° +- ПĐģĐ°ĐŗиĐŊ Đ´ĐģŅ ĐŋОддĐĩŅ€ĐļĐēи ĐŊĐĩŅĐēĐžĐģŅŒĐēиŅ… ĐŋĐžĐģŅŒĐˇĐžĐ˛Đ°Ņ‚ĐĩĐģĐĩĐš (openproxy) +- РайОŅ‚Đ° Ņ ĐģŅŽĐąŅ‹Đŧи ĐąŅ€Đ°ŅƒĐˇĐĩŅ€Đ°Đŧи и ĐžĐŋĐĩŅ€Đ°Ņ†Đ¸ĐžĐŊĐŊŅ‹Đŧи ŅĐ¸ŅŅ‚ĐĩĐŧĐ°Đŧи + +## ĐĸĐĩĐēŅƒŅ‰Đ¸Đĩ ĐžĐŗŅ€Đ°ĐŊиŅ‡ĐĩĐŊиŅ + +- ФаКĐģОвŅ‹Đĩ Ņ‚Ņ€Đ°ĐŊСаĐēŅ†Đ¸Đ¸ ĐŊĐĩ ŅĐļĐ°Ņ‚Ņ‹ +- НĐĩŅ‚ ĐŋŅ€Đ¸Đ˛Đ°Ņ‚ĐŊŅ‹Ņ… ŅĐ°ĐšŅ‚Ов + +## КаĐē ŅŅ‚Đž Ņ€Đ°ĐąĐžŅ‚Đ°ĐĩŅ‚? + +- ПоŅĐģĐĩ СаĐŋŅƒŅĐēĐ° `zeronet.py` вŅ‹ ŅĐŧĐžĐļĐĩŅ‚Đĩ ĐŋĐžŅĐĩŅ‰Đ°Ņ‚ŅŒ ŅĐ°ĐšŅ‚Ņ‹ в ZeroNet, иŅĐŋĐžĐģŅŒĐˇŅƒŅ Đ°Đ´Ņ€ĐĩŅ + `http://127.0.0.1:43110/{zeronet_Đ°Đ´Ņ€ĐĩŅ}` + (НаĐŋŅ€Đ¸ĐŧĐĩŅ€: `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`). +- КоĐŗĐ´Đ° вŅ‹ ĐŋĐžŅĐĩŅ‰Đ°ĐĩŅ‚Đĩ ĐŊОвŅ‹Đš ŅĐ°ĐšŅ‚ в ZeroNet, ĐžĐŊ ĐŋŅ‹Ņ‚Đ°ĐĩŅ‚ŅŅ ĐŊĐ°ĐšŅ‚и ĐŋиŅ€ĐžĐ˛ Ņ ĐŋĐžĐŧĐžŅ‰ŅŒŅŽ ĐŋŅ€ĐžŅ‚ĐžĐēĐžĐģĐ° BitTorrent, + Ņ‡Ņ‚ОйŅ‹ ŅĐēĐ°Ņ‡Đ°Ņ‚ŅŒ Ņƒ ĐŊиŅ… Ņ„Đ°ĐšĐģŅ‹ ŅĐ°ĐšŅ‚Đ° (HTML, CSS, JS и Ņ‚.Đ´.). +- ПоŅĐģĐĩ ĐŋĐžŅĐĩŅ‰ĐĩĐŊиŅ ŅĐ°ĐšŅ‚Đ° вŅ‹ Ņ‚ĐžĐļĐĩ ŅŅ‚Đ°ĐŊОвиŅ‚ĐĩŅŅŒ ĐĩĐŗĐž ĐŋиŅ€ĐžĐŧ. +- КаĐļĐ´Ņ‹Đš ŅĐ°ĐšŅ‚ ŅĐžĐ´ĐĩŅ€ĐļиŅ‚ Ņ„Đ°ĐšĐģ `content.json`, ĐēĐžŅ‚ĐžŅ€Ņ‹Đš ŅĐžĐ´ĐĩŅ€ĐļиŅ‚ SHA512 Ņ…ĐĩŅˆĐ¸ вŅĐĩŅ… ĐžŅŅ‚Đ°ĐģŅŒĐŊŅ‹Đĩ Ņ„Đ°ĐšĐģŅ‹ + и ĐŋОдĐŋиŅŅŒ, ŅĐžĐˇĐ´Đ°ĐŊĐŊŅƒŅŽ Ņ ĐŋĐžĐŧĐžŅ‰ŅŒŅŽ СаĐēŅ€Ņ‹Ņ‚ĐžĐŗĐž ĐēĐģŅŽŅ‡Đ° ŅĐ°ĐšŅ‚Đ°. +- ЕŅĐģи вĐģĐ°Đ´ĐĩĐģĐĩŅ† ŅĐ°ĐšŅ‚Đ° (Ņ‚ĐžŅ‚, ĐēŅ‚Đž вĐģĐ°Đ´ĐĩĐĩŅ‚ СаĐēŅ€Ņ‹Ņ‚Ņ‹Đŧ ĐēĐģŅŽŅ‡ĐžĐŧ Đ´ĐģŅ Đ°Đ´Ņ€ĐĩŅĐ° ŅĐ°ĐšŅ‚Đ°) иСĐŧĐĩĐŊŅĐĩŅ‚ ŅĐ°ĐšŅ‚, ĐžĐŊ + ĐŋОдĐŋиŅŅ‹Đ˛Đ°ĐĩŅ‚ ĐŊОвŅ‹Đš `content.json` и ĐŋŅƒĐąĐģиĐēŅƒĐĩŅ‚ ĐĩĐŗĐž Đ´ĐģŅ ĐŋиŅ€ĐžĐ˛. ПоŅĐģĐĩ ŅŅ‚ĐžĐŗĐž ĐŋиŅ€Ņ‹ ĐŋŅ€ĐžĐ˛ĐĩŅ€ŅŅŽŅ‚ Ņ†ĐĩĐģĐžŅŅ‚ĐŊĐžŅŅ‚ŅŒ `content.json` + (иŅĐŋĐžĐģŅŒĐˇŅƒŅ ĐŋОдĐŋиŅŅŒ), ŅĐēĐ°Ņ‡Đ˛Đ°ŅŽŅ‚ иСĐŧĐĩĐŊŅ‘ĐŊĐŊŅ‹Đĩ Ņ„Đ°ĐšĐģŅ‹ и Ņ€Đ°ŅĐŋŅ€ĐžŅŅ‚Ņ€Đ°ĐŊŅŅŽŅ‚ ĐŊОвŅ‹Đš ĐēĐžĐŊŅ‚ĐĩĐŊŅ‚ Đ´ĐģŅ Đ´Ņ€ŅƒĐŗиŅ… ĐŋиŅ€ĐžĐ˛. + +[ПŅ€ĐĩСĐĩĐŊŅ‚Đ°Ņ†Đ¸Ņ Đž ĐēŅ€Đ¸ĐŋŅ‚ĐžĐŗŅ€Đ°Ņ„ии ZeroNet, ОйĐŊОвĐģĐĩĐŊиŅŅ… ŅĐ°ĐšŅ‚Ов, ĐŧĐŊĐžĐŗĐžĐŋĐžĐģŅŒĐˇĐžĐ˛Đ°Ņ‚ĐĩĐģŅŒŅĐēиŅ… ŅĐ°ĐšŅ‚Đ°Ņ… Âģ](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000) +[ЧаŅŅ‚Đž СадаваĐĩĐŧŅ‹Đĩ вОĐŋŅ€ĐžŅŅ‹ Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) +[ДоĐēŅƒĐŧĐĩĐŊŅ‚Đ°Ņ†Đ¸Ņ Ņ€Đ°ĐˇŅ€Đ°ĐąĐžŅ‚Ņ‡Đ¸ĐēĐ° ZeroNet Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/) + +## ĐĄĐēŅ€Đ¸ĐŊŅˆĐžŅ‚Ņ‹ + +![Screenshot](https://i.imgur.com/H60OAHY.png) +![ZeroTalk](https://zeronet.io/docs/img/zerotalk.png) +[БоĐģŅŒŅˆĐĩ ŅĐēŅ€Đ¸ĐŊŅˆĐžŅ‚Ов в Đ´ĐžĐēŅƒĐŧĐĩĐŊŅ‚Đ°Ņ†Đ¸Đ¸ ZeroNet Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/) + +## КаĐē ĐŋŅ€Đ¸ŅĐžĐĩдиĐŊиŅ‚ŅŒŅŅ? + +### Windows + +- ĐĄĐēĐ°Ņ‡Đ°ĐšŅ‚Đĩ и Ņ€Đ°ŅĐŋĐ°ĐēŅƒĐšŅ‚Đĩ Đ°Ņ€Ņ…ив [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26МБ) +- ЗаĐŋŅƒŅŅ‚иŅ‚Đĩ `ZeroNet.exe` + +### macOS + +- ĐĄĐēĐ°Ņ‡Đ°ĐšŅ‚Đĩ и Ņ€Đ°ŅĐŋĐ°ĐēŅƒĐšŅ‚Đĩ Đ°Ņ€Ņ…ив [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14МБ) +- ЗаĐŋŅƒŅŅ‚иŅ‚Đĩ `ZeroNet.app` + +### Linux (64 йиŅ‚) + +- ĐĄĐēĐ°Ņ‡Đ°ĐšŅ‚Đĩ и Ņ€Đ°ŅĐŋĐ°ĐēŅƒĐšŅ‚Đĩ Đ°Ņ€Ņ…ив [ZeroNet-linux.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip) (14МБ) +- ЗаĐŋŅƒŅŅ‚иŅ‚Đĩ `./ZeroNet.sh` + +> **Note** +> ЗаĐŋŅƒŅŅ‚иŅ‚Đĩ Ņ‚Đ°ĐēиĐŧ ОйŅ€Đ°ĐˇĐžĐŧ: `./ZeroNet.sh --ui_ip '*' --ui_restrict ваŅˆ_ip_Đ°Đ´Ņ€ĐĩŅ`, Ņ‡Ņ‚ОйŅ‹ Ņ€Đ°ĐˇŅ€ĐĩŅˆĐ¸Ņ‚ŅŒ ŅƒĐ´Đ°ĐģŅ‘ĐŊĐŊĐžĐĩ ĐŋОдĐēĐģŅŽŅ‡ĐĩĐŊиĐĩ Đē вĐĩб–иĐŊŅ‚ĐĩŅ€Ņ„ĐĩĐšŅŅƒ. + +### Docker + +ОŅ„иŅ†Đ¸Đ°ĐģŅŒĐŊŅ‹Đš ОйŅ€Đ°Đˇ ĐŊĐ°Ņ…ОдиŅ‚ŅŅ СдĐĩŅŅŒ: https://hub.docker.com/r/canewsin/zeronet/ + +### Android (arm, arm64, x86) + +- ДĐģŅ Ņ€Đ°ĐąĐžŅ‚Ņ‹ Ņ‚Ņ€ĐĩĐąŅƒĐĩŅ‚ŅŅ Android ĐēĐ°Đē ĐŧиĐŊиĐŧŅƒĐŧ вĐĩŅ€ŅĐ¸Đ¸ 5.0 Lollipop +- [Download from Google Play](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile) +- ĐĄĐēĐ°Ņ‡Đ°Ņ‚ŅŒ APK: https://github.com/canewsin/zeronet_mobile/releases + +### Android (arm, arm64, x86) ОбĐģĐĩĐŗŅ‡Ņ‘ĐŊĐŊŅ‹Đš ĐēĐģиĐĩĐŊŅ‚ Ņ‚ĐžĐģŅŒĐēĐž Đ´ĐģŅ ĐŋŅ€ĐžŅĐŧĐžŅ‚Ņ€Đ° (1МБ) + +- ДĐģŅ Ņ€Đ°ĐąĐžŅ‚Ņ‹ Ņ‚Ņ€ĐĩĐąŅƒĐĩŅ‚ŅŅ Android ĐēĐ°Đē ĐŧиĐŊиĐŧŅƒĐŧ вĐĩŅ€ŅĐ¸Đ¸ 4.1 Jelly Bean +- [Download from Google Play](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite) + +### ĐŖŅŅ‚Đ°ĐŊОвĐēĐ° иС иŅŅ…ОдĐŊĐžĐŗĐž ĐēОда + +```sh +wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip +unzip ZeroNet-src.zip +cd ZeroNet +sudo apt-get update +sudo apt-get install python3-pip +sudo python3 -m pip install -r requirements.txt +``` +- ЗаĐŋŅƒŅŅ‚иŅ‚Đĩ `python3 zeronet.py` + +ОŅ‚ĐēŅ€ĐžĐšŅ‚Đĩ ĐŋŅ€Đ¸Đ˛ĐĩŅ‚ŅŅ‚вĐĩĐŊĐŊŅƒŅŽ ŅŅ‚Ņ€Đ°ĐŊиŅ†Ņƒ ZeroHello в ваŅˆĐĩĐŧ ĐąŅ€Đ°ŅƒĐˇĐĩŅ€Đĩ ĐŋĐž ŅŅŅ‹ĐģĐēĐĩ http://127.0.0.1:43110/ + +## КаĐē ĐŧĐŊĐĩ ŅĐžĐˇĐ´Đ°Ņ‚ŅŒ ŅĐ°ĐšŅ‚ в ZeroNet? + +- КĐģиĐēĐŊиŅ‚Đĩ ĐŊĐ° **⋮** > **"Create new, empty site"** в ĐŧĐĩĐŊŅŽ ĐŊĐ° ŅĐ°ĐšŅ‚Đĩ [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d). +- ВŅ‹ ĐąŅƒĐ´ĐĩŅ‚Đĩ **ĐŋĐĩŅ€ĐĩĐŊĐ°ĐŋŅ€Đ°Đ˛ĐģĐĩĐŊŅ‹** ĐŊĐ° ŅĐžĐ˛ĐĩŅ€ŅˆĐĩĐŊĐŊĐž ĐŊОвŅ‹Đš ŅĐ°ĐšŅ‚, ĐēĐžŅ‚ĐžŅ€Ņ‹Đš ĐŧĐžĐļĐĩŅ‚ ĐąŅ‹Ņ‚ŅŒ иСĐŧĐĩĐŊŅ‘ĐŊ Ņ‚ĐžĐģŅŒĐēĐž ваĐŧи! +- ВŅ‹ ĐŧĐžĐļĐĩŅ‚Đĩ ĐŊĐ°ĐšŅ‚и и иСĐŧĐĩĐŊиŅ‚ŅŒ ĐēĐžĐŊŅ‚ĐĩĐŊŅ‚ ваŅˆĐĩĐŗĐž ŅĐ°ĐšŅ‚Đ° в ĐēĐ°Ņ‚Đ°ĐģĐžĐŗĐĩ **data/[Đ°Đ´Ņ€ĐĩŅ_ваŅˆĐĩĐŗĐž_ŅĐ°ĐšŅ‚Đ°]** +- ПоŅĐģĐĩ иСĐŧĐĩĐŊĐĩĐŊиК ĐžŅ‚ĐēŅ€ĐžĐšŅ‚Đĩ ваŅˆ ŅĐ°ĐšŅ‚, ĐŋĐĩŅ€ĐĩĐēĐģŅŽŅ‡Đ¸Ņ‚Đĩ вĐģĐĩвО ĐēĐŊĐžĐŋĐēŅƒ "0" в ĐŋŅ€Đ°Đ˛ĐžĐŧ вĐĩŅ€Ņ…ĐŊĐĩĐŧ ŅƒĐŗĐģŅƒ, СаŅ‚ĐĩĐŧ ĐŊĐ°ĐļĐŧиŅ‚Đĩ ĐēĐŊĐžĐŋĐēи **sign** и **publish** вĐŊиСŅƒ + +ĐĄĐģĐĩĐ´ŅƒŅŽŅ‰Đ¸Đĩ ŅˆĐ°Đŗи: [ДоĐēŅƒĐŧĐĩĐŊŅ‚Đ°Ņ†Đ¸Ņ Ņ€Đ°ĐˇŅ€Đ°ĐąĐžŅ‚Ņ‡Đ¸ĐēĐ° ZeroNet](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/) + +## ПоддĐĩŅ€ĐļиŅ‚Đĩ ĐŋŅ€ĐžĐĩĐēŅ‚ + +- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Đ ĐĩĐēĐžĐŧĐĩĐŊĐ´ŅƒĐĩĐŧ) +- LiberaPay: https://liberapay.com/PramUkesh +- Paypal: https://paypal.me/PramUkesh +- ДŅ€ŅƒĐŗиĐĩ ŅĐŋĐžŅĐžĐąŅ‹: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive) + +#### ĐĄĐŋĐ°ŅĐ¸ĐąĐž! + +- ЗдĐĩŅŅŒ вŅ‹ ĐŧĐžĐļĐĩŅ‚Đĩ ĐŋĐžĐģŅƒŅ‡Đ¸Ņ‚ŅŒ йОĐģŅŒŅˆĐĩ иĐŊŅ„ĐžŅ€ĐŧĐ°Ņ†Đ¸Đ¸, ĐŋĐžĐŧĐžŅ‰ŅŒ, ĐŋŅ€ĐžŅ‡Đ¸Ņ‚Đ°Ņ‚ŅŒ ŅĐŋиŅĐžĐē иСĐŧĐĩĐŊĐĩĐŊиК и иŅŅĐģĐĩдОваŅ‚ŅŒ ZeroNet ŅĐ°ĐšŅ‚Ņ‹: https://www.reddit.com/r/zeronetx/ +- ОбŅ‰ĐĩĐŊиĐĩ ĐŋŅ€ĐžĐ¸ŅŅ…ОдиŅ‚ ĐŊĐ° ĐēĐ°ĐŊĐ°ĐģĐĩ [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) иĐģи в [Gitter](https://gitter.im/canewsin/ZeroNet) +- Đ­ĐģĐĩĐēŅ‚Ņ€ĐžĐŊĐŊĐ°Ņ ĐŋĐžŅ‡Ņ‚Đ°: canews.in@gmail.com diff --git a/README-zh-cn.md b/README-zh-cn.md new file mode 100644 index 00000000..37095ff6 --- /dev/null +++ b/README-zh-cn.md @@ -0,0 +1,132 @@ +# ZeroNet [![tests](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml/badge.svg)](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [![Docker Pulls](https://img.shields.io/docker/pulls/canewsin/zeronet)](https://hub.docker.com/r/canewsin/zeronet) + +[English](./README.md) + +äŊŋį”¨ Bitcoin 加密和 BitTorrent įŊ‘įģœįš„åŽģ中åŋƒåŒ–įŊ‘įģœ - https://zeronet.dev + + +## ä¸ēäģ€äšˆīŧŸ + +* 我äģŦį›¸äŋĄåŧ€æ”žīŧŒč‡Ēį”ąīŧŒæ— åŽĄæŸĨįš„įŊ‘įģœå’Œé€ščŽ¯ +* 不äŧšå—单į‚šæ•…éšœåŊąå“īŧšåĒčĻæœ‰åœ¨įēŋįš„节į‚šīŧŒįĢ™į‚šå°ąäŧšäŋæŒåœ¨įēŋ +* 无托įŽĄč´šį”¨īŧšįĢ™į‚šį”ąčŽŋé—Žč€…æ‰˜įŽĄ +* 无æŗ•å…ŗ闭īŧšå› ä¸ē节į‚šæ— å¤„不在 +* åŋĢ速åšļ可įĻģįēŋčŋčĄŒīŧšåŗäŊŋæ˛Ąæœ‰äē’联įŊ‘čŋžæŽĨ䚟可äģĨäŊŋį”¨ + + +## 功čƒŊ + * 厞æ—ļįĢ™į‚šæ›´æ–° + * 支持 Namecoin įš„ .bit 域名 + * 厉čŖ…æ–šäžŋīŧšåĒ需č§Ŗ压åšļčŋčĄŒ + * 一锎克隆存在įš„įĢ™į‚š + * 无需密į ã€åŸēäēŽ [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) + įš„čŽ¤č¯īŧšæ‚¨įš„č´ĻæˆˇčĸĢ与比į‰šå¸é’ąåŒ…į›¸åŒįš„加密斚æŗ•äŋæŠ¤ + * 内åģē SQL æœåŠĄå™¨å’Œ P2P 数捎同æ­ĨīŧščŽŠåŧ€å‘æ›´įŽ€å•åšļ提升加čŊŊ速åēĻ + * åŒŋ名性īŧšåŽŒæ•´įš„ Tor įŊ‘įģœæ”¯æŒīŧŒæ”¯æŒé€ščŋ‡ .onion éšč—æœåŠĄį›¸äē’čŋžæŽĨč€Œä¸æ˜¯é€ščŋ‡ IPv4 地址čŋžæŽĨ + * TLS 加密čŋžæŽĨ + * č‡Ē动打åŧ€ uPnP įĢ¯åŖ + * 多į”¨æˆˇīŧˆopenproxyīŧ‰æ”¯æŒįš„插äģļ + * 适į”¨äēŽäģģäŊ•æĩč§ˆå™¨ / 操äŊœįŗģįģŸ + + +## 原į† + +* 在čŋčĄŒ `zeronet.py` 后īŧŒæ‚¨å°†å¯äģĨ通čŋ‡ + `http://127.0.0.1:43110/{zeronet_address}`īŧˆäž‹åĻ‚īŧš + `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`īŧ‰čŽŋ问 zeronet 中įš„įĢ™į‚š +* 在您æĩč§ˆ zeronet įĢ™į‚šæ—ļīŧŒåŽĸæˆˇįĢ¯äŧšå°č¯•é€ščŋ‡ BitTorrent įŊ‘įģœæĨå¯ģ扞可į”¨įš„节į‚šīŧŒäģŽč€Œä¸‹čŊŊ需čĻįš„æ–‡äģļīŧˆhtmlīŧŒcssīŧŒjs...īŧ‰ +* 您将äŧšå‚¨å­˜æ¯ä¸€ä¸Ēæĩč§ˆčŋ‡įš„įĢ™į‚š +* 每ä¸ĒįĢ™į‚šéƒŊ包åĢ一ä¸Ē名ä¸ē `content.json` įš„æ–‡äģļīŧŒåŽƒå‚¨å­˜äē†å…ļäģ–所有文äģļįš„ sha512 æ•Ŗ列å€ŧäģĨ及一ä¸Ē通čŋ‡įĢ™į‚šį§é’Ĩį”Ÿæˆįš„į­žå +* åĻ‚æžœįĢ™į‚šįš„æ‰€æœ‰č€…īŧˆæ‹Ĩ有įĢ™į‚šåœ°å€įš„į§é’Ĩīŧ‰äŋŽæ”šäē†įĢ™į‚šīŧŒåšļ且äģ– / åĨšį­žåäē†æ–°įš„ `content.json` į„ļ后推送č‡ŗå…ļäģ–节į‚šīŧŒ + é‚Ŗ䚈čŋ™äē›čŠ‚į‚šå°†äŧšåœ¨äŊŋį”¨į­žåéĒŒč¯ `content.json` įš„įœŸåŽžæ€§åŽīŧŒä¸‹čŊŊäŋŽæ”šåŽįš„æ–‡äģļåšļ将新内厚推送č‡ŗåĻ外įš„节į‚š + +#### [å…ŗäēŽ ZeroNet 加密īŧŒįĢ™į‚šæ›´æ–°īŧŒå¤šį”¨æˆˇįĢ™į‚šįš„åšģį¯į‰‡ Âģ](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000) +#### [å¸¸č§é—Žéĸ˜ Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) + +#### [ZeroNet åŧ€å‘č€…æ–‡æĄŖ Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/) + + +## åąåš•æˆĒ回 + +![Screenshot](https://i.imgur.com/H60OAHY.png) +![ZeroTalk](https://zeronet.io/docs/img/zerotalk.png) + +#### [ZeroNet 文æĄŖ中įš„æ›´å¤šåąåš•æˆĒ回 Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/) + + +## åĻ‚äŊ•åŠ å…Ĩ + +### Windows + + - 下čŊŊ [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26MB) + - 在äģģ意äŊįŊŽč§Ŗ压įŧŠ + - čŋčĄŒ `ZeroNet.exe` + +### macOS + + - 下čŊŊ [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14MB) + - 在äģģ意äŊįŊŽč§Ŗ压įŧŠ + - čŋčĄŒ `ZeroNet.app` + +### Linux (x86-64bit) + + - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip` + - `unzip ZeroNet-linux.zip` + - `cd ZeroNet-linux` + - äŊŋį”¨äģĨ下å‘Ŋäģ¤å¯åŠ¨ `./ZeroNet.sh` + - 在æĩč§ˆå™¨æ‰“åŧ€ http://127.0.0.1:43110/ åŗ可čŽŋ问 ZeroHello éĄĩéĸ + + __提į¤ēīŧš__ č‹ĨčĻå…čŽ¸åœ¨ Web į•Œéĸ上įš„čŋœį¨‹čŋžæŽĨīŧŒäŊŋį”¨äģĨ下å‘Ŋäģ¤å¯åŠ¨ `./ZeroNet.sh --ui_ip '*' --ui_restrict your.ip.address` + +### äģŽæēäģŖį åŽ‰čŖ… + + - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip` + - `unzip ZeroNet-src.zip` + - `cd ZeroNet` + - `sudo apt-get update` + - `sudo apt-get install python3-pip` + - `sudo python3 -m pip install -r requirements.txt` + - äŊŋį”¨äģĨ下å‘Ŋäģ¤å¯åŠ¨ `python3 zeronet.py` + - 在æĩč§ˆå™¨æ‰“åŧ€ http://127.0.0.1:43110/ åŗ可čŽŋ问 ZeroHello éĄĩéĸ + + ### Android (arm, arm64, x86) + - minimum Android version supported 21 (Android 5.0 Lollipop) + - [Download from Google Play](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile) + - APK download: https://github.com/canewsin/zeronet_mobile/releases + +### Android (arm, arm64, x86) Thin Client for Preview Only (Size 1MB) + - minimum Android version supported 16 (JellyBean) + - [Download from Google Play](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite) + +## įŽ°æœ‰é™åˆļ + +* äŧ čž“æ–‡äģļæ—ļæ˛Ąæœ‰åŽ‹įŧŠ +* 不支持į§æœ‰įĢ™į‚š + + +## åĻ‚äŊ•åˆ›åģē一ä¸Ē ZeroNet įĢ™į‚šīŧŸ + + * į‚šå‡ģ [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d) įĢ™į‚šįš„ **⋮** > **「新åģēįŠēįĢ™į‚šã€** čœå•éĄš + * 您将čĸĢ**重厚向**到一ä¸Ē全新įš„įĢ™į‚šīŧŒč¯ĨįĢ™į‚šåĒčƒŊį”ąæ‚¨äŋŽæ”š + * 您可äģĨ在 **data/[您įš„įĢ™į‚šåœ°å€]** į›ŽåŊ•ä¸­æ‰žåˆ°åšļäŋŽæ”šįŊ‘įĢ™įš„内厚 + * äŋŽæ”šåŽæ‰“åŧ€æ‚¨įš„įŊ‘įĢ™īŧŒå°†åŗ上角įš„「0」按钎拖到åˇĻ䞧īŧŒį„ļ后į‚šå‡ģåē•éƒ¨įš„**į­žå**åšļ**发布**按钎 + +æŽĨ下æĨįš„æ­ĨéĒ¤īŧš[ZeroNet åŧ€å‘č€…æ–‡æĄŖ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/) + +## 帎劊čŋ™ä¸Ē饚į›Ž +- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Preferred) +- LiberaPay: https://liberapay.com/PramUkesh +- Paypal: https://paypal.me/PramUkesh +- Others: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive) + + +#### 感č°ĸ您īŧ + +* 更多äŋĄæ¯īŧŒå¸ŽåŠŠīŧŒå˜æ›´čŽ°åŊ•å’Œ zeronet įĢ™į‚šīŧšhttps://www.reddit.com/r/zeronetx/ +* 前垀 [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) 或 [gitter](https://gitter.im/canewsin/ZeroNet) 和我äģŦčŠå¤Š +* [čŋ™é‡Œ](https://gitter.im/canewsin/ZeroNet)是一ä¸Ē gitter 上įš„ä¸­æ–‡čŠå¤ŠåŽ¤ +* Email: canews.in@gmail.com diff --git a/README.md b/README.md new file mode 100644 index 00000000..70b79adc --- /dev/null +++ b/README.md @@ -0,0 +1,156 @@ +# ZeroNet [![tests](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml/badge.svg)](https://github.com/ZeroNetX/ZeroNet/actions/workflows/tests.yml) [![Documentation](https://img.shields.io/badge/docs-faq-brightgreen.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) [![Help](https://img.shields.io/badge/keep_this_project_alive-donate-yellow.svg)](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/) [![Docker Pulls](https://img.shields.io/docker/pulls/canewsin/zeronet)](https://hub.docker.com/r/canewsin/zeronet) + +Decentralized websites using Bitcoin crypto and the BitTorrent network - https://zeronet.dev / [ZeroNet Site](http://127.0.0.1:43110/1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX/), Unlike Bitcoin, ZeroNet Doesn't need a blockchain to run, But uses cryptography used by BTC, to ensure data integrity and validation. + + +## Why? + +* We believe in open, free, and uncensored network and communication. +* No single point of failure: Site remains online so long as at least 1 peer is + serving it. +* No hosting costs: Sites are served by visitors. +* Impossible to shut down: It's nowhere because it's everywhere. +* Fast and works offline: You can access the site even if Internet is + unavailable. + + +## Features + * Real-time updated sites + * Namecoin .bit domains support + * Easy to setup: unpack & run + * Clone websites in one click + * Password-less [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) + based authorization: Your account is protected by the same cryptography as your Bitcoin wallet + * Built-in SQL server with P2P data synchronization: Allows easier site development and faster page load times + * Anonymity: Full Tor network support with .onion hidden services instead of IPv4 addresses + * TLS encrypted connections + * Automatic uPnP port opening + * Plugin for multiuser (openproxy) support + * Works with any browser/OS + + +## How does it work? + +* After starting `zeronet.py` you will be able to visit zeronet sites using + `http://127.0.0.1:43110/{zeronet_address}` (eg. + `http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d`). +* When you visit a new zeronet site, it tries to find peers using the BitTorrent + network so it can download the site files (html, css, js...) from them. +* Each visited site is also served by you. +* Every site contains a `content.json` file which holds all other files in a sha512 hash + and a signature generated using the site's private key. +* If the site owner (who has the private key for the site address) modifies the + site and signs the new `content.json` and publishes it to the peers. + Afterwards, the peers verify the `content.json` integrity (using the + signature), they download the modified files and publish the new content to + other peers. + +#### [Slideshow about ZeroNet cryptography, site updates, multi-user sites Âģ](https://docs.google.com/presentation/d/1_2qK1IuOKJ51pgBvllZ9Yu7Au2l551t3XBgyTSvilew/pub?start=false&loop=false&delayms=3000) +#### [Frequently asked questions Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/faq/) + +#### [ZeroNet Developer Documentation Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/) + + +## Screenshots + +![Screenshot](https://i.imgur.com/H60OAHY.png) +![ZeroTalk](https://zeronet.io/docs/img/zerotalk.png) + +#### [More screenshots in ZeroNet docs Âģ](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/using_zeronet/sample_sites/) + + +## How to join + +### Windows + + - Download [ZeroNet-win.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-win.zip) (26MB) + - Unpack anywhere + - Run `ZeroNet.exe` + +### macOS + + - Download [ZeroNet-mac.zip](https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-mac.zip) (14MB) + - Unpack anywhere + - Run `ZeroNet.app` + +### Linux (x86-64bit) + - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-linux.zip` + - `unzip ZeroNet-linux.zip` + - `cd ZeroNet-linux` + - Start with: `./ZeroNet.sh` + - Open the ZeroHello landing page in your browser by navigating to: http://127.0.0.1:43110/ + + __Tip:__ Start with `./ZeroNet.sh --ui_ip '*' --ui_restrict your.ip.address` to allow remote connections on the web interface. + + ### Android (arm, arm64, x86) + - minimum Android version supported 21 (Android 5.0 Lollipop) + - [Download from Google Play](https://play.google.com/store/apps/details?id=in.canews.zeronetmobile) + - APK download: https://github.com/canewsin/zeronet_mobile/releases + +### Android (arm, arm64, x86) Thin Client for Preview Only (Size 1MB) + - minimum Android version supported 16 (JellyBean) + - [Download from Google Play](https://play.google.com/store/apps/details?id=dev.zeronetx.app.lite) + + +#### Docker +There is an official image, built from source at: https://hub.docker.com/r/canewsin/zeronet/ + +### Online Proxies +Proxies are like seed boxes for sites(i.e ZNX runs on a cloud vps), you can try zeronet experience from proxies. Add your proxy below if you have one. + +#### Official ZNX Proxy : + +https://proxy.zeronet.dev/ + +https://zeronet.dev/ + +#### From Community + +https://0net-preview.com/ + +https://portal.ngnoid.tv/ + +https://zeronet.ipfsscan.io/ + + +### Install from source + + - `wget https://github.com/ZeroNetX/ZeroNet/releases/latest/download/ZeroNet-src.zip` + - `unzip ZeroNet-src.zip` + - `cd ZeroNet` + - `sudo apt-get update` + - `sudo apt-get install python3-pip` + - `sudo python3 -m pip install -r requirements.txt` + - Start with: `python3 zeronet.py` + - Open the ZeroHello landing page in your browser by navigating to: http://127.0.0.1:43110/ + +## Current limitations + +* File transactions are not compressed +* No private sites + + +## How can I create a ZeroNet site? + + * Click on **⋮** > **"Create new, empty site"** menu item on the site [ZeroHello](http://127.0.0.1:43110/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d). + * You will be **redirected** to a completely new site that is only modifiable by you! + * You can find and modify your site's content in **data/[yoursiteaddress]** directory + * After the modifications open your site, drag the topright "0" button to left, then press **sign** and **publish** buttons on the bottom + +Next steps: [ZeroNet Developer Documentation](https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/site_development/getting_started/) + +## Help keep this project alive +- Bitcoin: 1ZeroNetyV5mKY9JF1gsm82TuBXHpfdLX (Preferred) +- LiberaPay: https://liberapay.com/PramUkesh +- Paypal: https://paypal.me/PramUkesh +- Others: [Donate](!https://docs.zeronet.dev/1DeveLopDZL1cHfKi8UXHh2UBEhzH6HhMp/help_zeronet/donate/#help-to-keep-zeronet-development-alive) + +#### Thank you! + +* More info, help, changelog, zeronet sites: https://www.reddit.com/r/zeronetx/ +* Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) or on [gitter](https://gitter.im/canewsin/ZeroNet) +* Email: canews.in@gmail.com diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..24fe0c45 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,45 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + #Set box + config.vm.box = "ubuntu/trusty64" + + #Do not check fo updates + config.vm.box_check_update = false + + #Add private network + config.vm.network "private_network", type: "dhcp" + + #Redirect ports + config.vm.network "forwarded_port", guest: 43110, host: 43110 + config.vm.network "forwarded_port", guest: 15441, host: 15441 + + #Sync folder using NFS if not windows + config.vm.synced_folder ".", "/vagrant", + :nfs => !Vagrant::Util::Platform.windows? + + #Virtal Box settings + config.vm.provider "virtualbox" do |vb| + # Don't boot with headless mode + #vb.gui = true + + # Set VM settings + vb.customize ["modifyvm", :id, "--memory", "512"] + vb.customize ["modifyvm", :id, "--cpus", 1] + end + + #Update system + config.vm.provision "shell", + inline: "sudo apt-get update -y && sudo apt-get upgrade -y" + + #Install deps + config.vm.provision "shell", + inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y" + config.vm.provision "shell", + inline: "sudo pip install msgpack --upgrade" + +end diff --git a/md5.hashes b/md5.hashes deleted file mode 100644 index c11c8274..00000000 --- a/md5.hashes +++ /dev/null @@ -1,34 +0,0 @@ -[ - "794f5ac0675f66310963163e62527196", - "f4fdc4ef9fcf3db65ea91fb46b3982ca", - "4bdd9cc3fd3629a7e177bf37df5326c6", - "3df0aae9c0f30941a3893f02b0533d65", - "25001a7ef26550ec1fbb2ae7fbfff6a1", - "634647a7ea916b29f3a8fe5f140341a8", - "e09fab4484cf10d5bc29901f5c17df78", - "11af969820fdc72db9d9c41abd98e4c9", - "371da38ccd0dcdc49b71edd0872be41e", - "a23aeb4308119a2e34e33c109d4ee496", - "0386c7231f8af2706f3b8ca71bb30a82", - "0f408bbceb7572631b0e1dcd97b257e1", - "d4cfb19351a761ae1252934357772f1e", - "7656733d355d0a31ee57ba3901374de8", - "b522f9ad4d17d8962bba7fc1c6880d1a", - "3e8dab64ea8c23463f83de1c68bc2342", - "b5ebbd8c4a7fa865095e95853d5bee35", - "0e7b811892a6abc0cbcf66161ac82bc5", - "d2ba546cd3eae258b10c7fdbaafe9434", - "f558010cc964e206eb03eafd90731e0b", - "4cfcd90b9206701d96c7757222072e5c", - "063cd806f972b6d0f0226d8c04474270", - "c7d737758baf1d516cf3a0ed45176f6e", - "b6cfb932d1499cbc2fba10c06efe9567", - "30865832830c3bb1d67aeb48b0572774", - "4908d51ff8f2daa35a209db0c86dc535", - "336b451616f620743e6aecb30900b822", - "98c9109d618094a9775866c1838d4666", - "11e86b9a2aae72f854bf1f181946d78b", - "28d0faceb156ad1e5f1befa770dce3cd", - "93191cea5d81f6c2b2f5a4a547e2bdfd", - "6b1f09c95720e730ef27970b7f9f3e5c" -] diff --git a/patches.json b/patches.json deleted file mode 100644 index 54e91f51..00000000 --- a/patches.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "filename": "CryptConnection.py", - "patchDir": "src/Crypt", - "patchUrl": "https://raw.githubusercontent.com/canewsin/ZeroNet/py3-patches/CryptConnection.py" - } -] \ No newline at end of file diff --git a/plugins b/plugins new file mode 160000 index 00000000..689d9309 --- /dev/null +++ b/plugins @@ -0,0 +1 @@ +Subproject commit 689d9309f73371f4681191b125ec3f2e14075eeb diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..538a6dfc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +gevent==1.4.0; python_version <= "3.6" +greenlet==0.4.16; python_version <= "3.6" +gevent>=20.9.0; python_version >= "3.7" +msgpack>=0.4.4 +base58 +merkletools @ git+https://github.com/ZeroNetX/pymerkletools.git@dev +rsa +PySocks>=1.6.8 +pyasn1 +websocket_client +gevent-ws +coincurve +maxminddb diff --git a/src/Config.py b/src/Config.py new file mode 100644 index 00000000..a9208d55 --- /dev/null +++ b/src/Config.py @@ -0,0 +1,675 @@ +import argparse +import sys +import os +import locale +import re +import configparser +import logging +import logging.handlers +import stat +import time + + +class Config(object): + + def __init__(self, argv): + self.version = "0.9.0" + self.rev = 4630 + self.argv = argv + self.action = None + self.test_parser = None + self.pending_changes = {} + self.need_restart = False + self.keys_api_change_allowed = set([ + "tor", "fileserver_port", "language", "tor_use_bridges", "trackers_proxy", "trackers", + "trackers_file", "open_browser", "log_level", "fileserver_ip_type", "ip_external", "offline", + "threads_fs_read", "threads_fs_write", "threads_crypt", "threads_db" + ]) + self.keys_restart_need = set([ + "tor", "fileserver_port", "fileserver_ip_type", "threads_fs_read", "threads_fs_write", "threads_crypt", "threads_db" + ]) + self.start_dir = self.getStartDir() + + self.config_file = self.start_dir + "/zeronet.conf" + self.data_dir = self.start_dir + "/data" + self.log_dir = self.start_dir + "/log" + self.openssl_lib_file = None + self.openssl_bin_file = None + + self.trackers_file = False + self.createParser() + self.createArguments() + + def createParser(self): + # Create parser + self.parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + self.parser.register('type', 'bool', self.strToBool) + self.subparsers = self.parser.add_subparsers(title="Action to perform", dest="action") + + def __str__(self): + return str(self.arguments).replace("Namespace", "Config") # Using argparse str output + + # Convert string to bool + def strToBool(self, v): + return v.lower() in ("yes", "true", "t", "1") + + def getStartDir(self): + this_file = os.path.abspath(__file__).replace("\\", "/").rstrip("cd") + + if "--start_dir" in self.argv: + start_dir = self.argv[self.argv.index("--start_dir") + 1] + elif this_file.endswith("/Contents/Resources/core/src/Config.py"): + # Running as ZeroNet.app + if this_file.startswith("/Application") or this_file.startswith("/private") or this_file.startswith(os.path.expanduser("~/Library")): + # Runnig from non-writeable directory, put data to Application Support + start_dir = os.path.expanduser("~/Library/Application Support/ZeroNet") + else: + # Running from writeable directory put data next to .app + start_dir = re.sub("/[^/]+/Contents/Resources/core/src/Config.py", "", this_file) + elif this_file.endswith("/core/src/Config.py"): + # Running as exe or source is at Application Support directory, put var files to outside of core dir + start_dir = this_file.replace("/core/src/Config.py", "") + elif this_file.endswith("usr/share/zeronet/src/Config.py"): + # Running from non-writeable location, e.g., AppImage + start_dir = os.path.expanduser("~/ZeroNet") + else: + start_dir = "." + + return start_dir + + # Create command line arguments + def createArguments(self): + from Crypt import CryptHash + access_key_default = CryptHash.random(24, "base64") # Used to allow restrited plugins when multiuser plugin is enabled + trackers = [ + "http://open.acgnxtracker.com:80/announce", # DE + "http://tracker.bt4g.com:2095/announce", # Cloudflare + "http://tracker.files.fm:6969/announce", + "http://t.publictracker.xyz:6969/announce", + "https://tracker.lilithraws.cf:443/announce", + "https://tracker.babico.name.tr:443/announce", + ] + # Platform specific + if sys.platform.startswith("win"): + coffeescript = "type %s | tools\\coffee\\coffee.cmd" + else: + coffeescript = None + + try: + language, enc = locale.getdefaultlocale() + language = language.lower().replace("_", "-") + if language not in ["pt-br", "zh-tw"]: + language = language.split("-")[0] + except Exception: + language = "en" + + use_openssl = True + + if repr(1483108852.565) != "1483108852.565": # Fix for weird Android issue + fix_float_decimals = True + else: + fix_float_decimals = False + + config_file = self.start_dir + "/zeronet.conf" + data_dir = self.start_dir + "/data" + log_dir = self.start_dir + "/log" + + ip_local = ["127.0.0.1", "::1"] + + # Main + action = self.subparsers.add_parser("main", help='Start UiServer and FileServer (default)') + + # SiteCreate + action = self.subparsers.add_parser("siteCreate", help='Create a new site') + action.register('type', 'bool', self.strToBool) + action.add_argument('--use_master_seed', help="Allow created site's private key to be recovered using the master seed in users.json (default: True)", type="bool", choices=[True, False], default=True) + + # SiteNeedFile + action = self.subparsers.add_parser("siteNeedFile", help='Get a file from site') + action.add_argument('address', help='Site address') + action.add_argument('inner_path', help='File inner path') + + # SiteDownload + action = self.subparsers.add_parser("siteDownload", help='Download a new site') + action.add_argument('address', help='Site address') + + # SiteSign + action = self.subparsers.add_parser("siteSign", help='Update and sign content.json: address [privatekey]') + action.add_argument('address', help='Site to sign') + action.add_argument('privatekey', help='Private key (default: ask on execute)', nargs='?') + action.add_argument('--inner_path', help='File you want to sign (default: content.json)', + default="content.json", metavar="inner_path") + action.add_argument('--remove_missing_optional', help='Remove optional files that is not present in the directory', action='store_true') + action.add_argument('--publish', help='Publish site after the signing', action='store_true') + + # SitePublish + action = self.subparsers.add_parser("sitePublish", help='Publish site to other peers: address') + action.add_argument('address', help='Site to publish') + action.add_argument('peer_ip', help='Peer ip to publish (default: random peers ip from tracker)', + default=None, nargs='?') + action.add_argument('peer_port', help='Peer port to publish (default: random peer port from tracker)', + default=15441, nargs='?') + action.add_argument('--inner_path', help='Content.json you want to publish (default: content.json)', + default="content.json", metavar="inner_path") + + # SiteVerify + action = self.subparsers.add_parser("siteVerify", help='Verify site files using sha512: address') + action.add_argument('address', help='Site to verify') + + # SiteCmd + action = self.subparsers.add_parser("siteCmd", help='Execute a ZeroFrame API command on a site') + action.add_argument('address', help='Site address') + action.add_argument('cmd', help='API command name') + action.add_argument('parameters', help='Parameters of the command', nargs='?') + + # dbRebuild + action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache') + action.add_argument('address', help='Site to rebuild') + + # dbQuery + action = self.subparsers.add_parser("dbQuery", help='Query site sql cache') + action.add_argument('address', help='Site to query') + action.add_argument('query', help='Sql query') + + # PeerPing + action = self.subparsers.add_parser("peerPing", help='Send Ping command to peer') + action.add_argument('peer_ip', help='Peer ip') + action.add_argument('peer_port', help='Peer port', nargs='?') + + # PeerGetFile + action = self.subparsers.add_parser("peerGetFile", help='Request and print a file content from peer') + action.add_argument('peer_ip', help='Peer ip') + action.add_argument('peer_port', help='Peer port') + action.add_argument('site', help='Site address') + action.add_argument('filename', help='File name to request') + action.add_argument('--benchmark', help='Request file 10x then displays the total time', action='store_true') + + # PeerCmd + action = self.subparsers.add_parser("peerCmd", help='Request and print a file content from peer') + action.add_argument('peer_ip', help='Peer ip') + action.add_argument('peer_port', help='Peer port') + action.add_argument('cmd', help='Command to execute') + action.add_argument('parameters', help='Parameters to command', nargs='?') + + # CryptSign + action = self.subparsers.add_parser("cryptSign", help='Sign message using Bitcoin private key') + action.add_argument('message', help='Message to sign') + action.add_argument('privatekey', help='Private key') + + # Crypt Verify + action = self.subparsers.add_parser("cryptVerify", help='Verify message using Bitcoin public address') + action.add_argument('message', help='Message to verify') + action.add_argument('sign', help='Signiture for message') + action.add_argument('address', help='Signer\'s address') + + # Crypt GetPrivatekey + action = self.subparsers.add_parser("cryptGetPrivatekey", help='Generate a privatekey from master seed') + action.add_argument('master_seed', help='Source master seed') + action.add_argument('site_address_index', help='Site address index', type=int) + + action = self.subparsers.add_parser("getConfig", help='Return json-encoded info') + action = self.subparsers.add_parser("testConnection", help='Testing') + action = self.subparsers.add_parser("testAnnounce", help='Testing') + + self.test_parser = self.subparsers.add_parser("test", help='Run a test') + self.test_parser.add_argument('test_name', help='Test name', nargs="?") + # self.test_parser.add_argument('--benchmark', help='Run the tests multiple times to measure the performance', action='store_true') + + # Config parameters + self.parser.add_argument('--verbose', help='More detailed logging', action='store_true') + self.parser.add_argument('--debug', help='Debug mode', action='store_true') + self.parser.add_argument('--silent', help='Only log errors to terminal output', action='store_true') + self.parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') + self.parser.add_argument('--merge_media', help='Merge all.js and all.css', action='store_true') + + self.parser.add_argument('--batch', help="Batch mode (No interactive input for commands)", action='store_true') + + self.parser.add_argument('--start_dir', help='Path of working dir for variable content (data, log, .conf)', default=self.start_dir, metavar="path") + self.parser.add_argument('--config_file', help='Path of config file', default=config_file, metavar="path") + self.parser.add_argument('--data_dir', help='Path of data directory', default=data_dir, metavar="path") + + self.parser.add_argument('--console_log_level', help='Level of logging to console', default="default", choices=["default", "DEBUG", "INFO", "ERROR", "off"]) + + self.parser.add_argument('--log_dir', help='Path of logging directory', default=log_dir, metavar="path") + self.parser.add_argument('--log_level', help='Level of logging to file', default="DEBUG", choices=["DEBUG", "INFO", "ERROR", "off"]) + self.parser.add_argument('--log_rotate', help='Log rotate interval', default="daily", choices=["hourly", "daily", "weekly", "off"]) + self.parser.add_argument('--log_rotate_backup_count', help='Log rotate backup count', default=5, type=int) + + self.parser.add_argument('--language', help='Web interface language', default=language, metavar='language') + self.parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') + self.parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') + self.parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*') + self.parser.add_argument('--ui_host', help='Allow access using this hosts', metavar='host', nargs='*') + self.parser.add_argument('--ui_trans_proxy', help='Allow access using a transparent proxy', action='store_true') + + self.parser.add_argument('--open_browser', help='Open homepage in web browser automatically', + nargs='?', const="default_browser", metavar='browser_name') + self.parser.add_argument('--homepage', help='Web interface Homepage', default='1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d', + metavar='address') + self.parser.add_argument('--updatesite', help='Source code update site', default='1Update8crprmciJHwp2WXqkx2c4iYp18', + metavar='address') + self.parser.add_argument('--access_key', help='Plugin access key default: Random key generated at startup', default=access_key_default, metavar='key') + self.parser.add_argument('--dist_type', help='Type of installed distribution', default='source') + + self.parser.add_argument('--size_limit', help='Default site size limit in MB', default=25, type=int, metavar='limit') + self.parser.add_argument('--file_size_limit', help='Maximum per file size limit in MB', default=10, type=int, metavar='limit') + self.parser.add_argument('--connected_limit', help='Max connected peer per site', default=8, type=int, metavar='connected_limit') + self.parser.add_argument('--global_connected_limit', help='Max connections', default=512, type=int, metavar='global_connected_limit') + self.parser.add_argument('--workers', help='Download workers per site', default=5, type=int, metavar='workers') + + self.parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') + self.parser.add_argument('--fileserver_port', help='FileServer bind port (0: randomize)', default=0, type=int, metavar='port') + self.parser.add_argument('--fileserver_port_range', help='FileServer randomization range', default="10000-40000", metavar='port') + self.parser.add_argument('--fileserver_ip_type', help='FileServer ip type', default="dual", choices=["ipv4", "ipv6", "dual"]) + self.parser.add_argument('--ip_local', help='My local ips', default=ip_local, type=int, metavar='ip', nargs='*') + self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip', nargs='*') + self.parser.add_argument('--offline', help='Disable network communication', action='store_true') + + self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') + self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') + self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip') + self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') + self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*') + self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable") + self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True) + self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=True) + self.parser.add_argument('--openssl_lib_file', help='Path for OpenSSL library file (default: detect)', default=argparse.SUPPRESS, metavar="path") + self.parser.add_argument('--openssl_bin_file', help='Path for OpenSSL binary file (default: detect)', default=argparse.SUPPRESS, metavar="path") + self.parser.add_argument('--disable_db', help='Disable database updating', action='store_true') + self.parser.add_argument('--disable_encryption', help='Disable connection encryption', action='store_true') + self.parser.add_argument('--force_encryption', help="Enforce encryption to all peer connections", action='store_true') + self.parser.add_argument('--disable_sslcompression', help='Disable SSL compression to save memory', + type='bool', choices=[True, False], default=True) + self.parser.add_argument('--keep_ssl_cert', help='Disable new SSL cert generation on startup', action='store_true') + self.parser.add_argument('--max_files_opened', help='Change maximum opened files allowed by OS to this value on startup', + default=2048, type=int, metavar='limit') + self.parser.add_argument('--stack_size', help='Change thread stack size', default=None, type=int, metavar='thread_stack_size') + self.parser.add_argument('--use_tempfiles', help='Use temporary files when downloading (experimental)', + type='bool', choices=[True, False], default=False) + self.parser.add_argument('--stream_downloads', help='Stream download directly to files (experimental)', + type='bool', choices=[True, False], default=False) + self.parser.add_argument("--msgpack_purepython", help='Use less memory, but a bit more CPU power', + type='bool', choices=[True, False], default=False) + self.parser.add_argument("--fix_float_decimals", help='Fix content.json modification date float precision on verification', + type='bool', choices=[True, False], default=fix_float_decimals) + self.parser.add_argument("--db_mode", choices=["speed", "security"], default="speed") + + self.parser.add_argument('--threads_fs_read', help='Number of threads for file read operations', default=1, type=int) + self.parser.add_argument('--threads_fs_write', help='Number of threads for file write operations', default=1, type=int) + self.parser.add_argument('--threads_crypt', help='Number of threads for cryptographic operations', default=2, type=int) + self.parser.add_argument('--threads_db', help='Number of threads for database operations', default=1, type=int) + + self.parser.add_argument("--download_optional", choices=["manual", "auto"], default="manual") + + self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, + metavar='executable_path') + + self.parser.add_argument('--tor', help='enable: Use only for Tor peers, always: Use Tor for every connection', choices=["disable", "enable", "always"], default='enable') + self.parser.add_argument('--tor_controller', help='Tor controller address', metavar='ip:port', default='127.0.0.1:9051') + self.parser.add_argument('--tor_proxy', help='Tor proxy address', metavar='ip:port', default='127.0.0.1:9050') + self.parser.add_argument('--tor_password', help='Tor controller password', metavar='password') + self.parser.add_argument('--tor_use_bridges', help='Use obfuscated bridge relays to avoid Tor block', action='store_true') + self.parser.add_argument('--tor_hs_limit', help='Maximum number of hidden services in Tor always mode', metavar='limit', type=int, default=10) + self.parser.add_argument('--tor_hs_port', help='Hidden service port in Tor always mode', metavar='limit', type=int, default=15441) + + self.parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev)) + self.parser.add_argument('--end', help='Stop multi value argument parsing', action='store_true') + + return self.parser + + def loadTrackersFile(self): + if not self.trackers_file: + self.trackers_file = ["trackers.txt", "{data_dir}/1HELLoE3sFD9569CLCbHEAVqvqV7U2Ri9d/trackers.txt"] + self.trackers = self.arguments.trackers[:] + + for trackers_file in self.trackers_file: + try: + if trackers_file.startswith("/"): # Absolute + trackers_file_path = trackers_file + elif trackers_file.startswith("{data_dir}"): # Relative to data_dir + trackers_file_path = trackers_file.replace("{data_dir}", self.data_dir) + else: # Relative to zeronet.py + trackers_file_path = self.start_dir + "/" + trackers_file + + if not os.path.exists(trackers_file_path): + continue + + for line in open(trackers_file_path): + tracker = line.strip() + if "://" in tracker and tracker not in self.trackers: + self.trackers.append(tracker) + except Exception as err: + print("Error loading trackers file: %s" % err) + + # Find arguments specified for current action + def getActionArguments(self): + back = {} + arguments = self.parser._subparsers._group_actions[0].choices[self.action]._actions[1:] # First is --version + for argument in arguments: + back[argument.dest] = getattr(self, argument.dest) + return back + + # Try to find action from argv + def getAction(self, argv): + actions = [list(action.choices.keys()) for action in self.parser._actions if action.dest == "action"][0] # Valid actions + found_action = False + for action in actions: # See if any in argv + if action in argv: + found_action = action + break + return found_action + + # Move plugin parameters to end of argument list + def moveUnknownToEnd(self, argv, default_action): + valid_actions = sum([action.option_strings for action in self.parser._actions], []) + valid_parameters = [] + plugin_parameters = [] + plugin = False + for arg in argv: + if arg.startswith("--"): + if arg not in valid_actions: + plugin = True + else: + plugin = False + elif arg == default_action: + plugin = False + + if plugin: + plugin_parameters.append(arg) + else: + valid_parameters.append(arg) + return valid_parameters + plugin_parameters + + def getParser(self, argv): + action = self.getAction(argv) + if not action: + return self.parser + else: + return self.subparsers.choices[action] + + # Parse arguments from config file and command line + def parse(self, silent=False, parse_config=True): + argv = self.argv[:] # Copy command line arguments + current_parser = self.getParser(argv) + if silent: # Don't display messages or quit on unknown parameter + original_print_message = self.parser._print_message + original_exit = self.parser.exit + + def silencer(parser, function_name): + parser.exited = True + return None + current_parser.exited = False + current_parser._print_message = lambda *args, **kwargs: silencer(current_parser, "_print_message") + current_parser.exit = lambda *args, **kwargs: silencer(current_parser, "exit") + + self.parseCommandline(argv, silent) # Parse argv + self.setAttributes() + if parse_config: + argv = self.parseConfig(argv) # Add arguments from config file + + self.parseCommandline(argv, silent) # Parse argv + self.setAttributes() + + if not silent: + if self.fileserver_ip != "*" and self.fileserver_ip not in self.ip_local: + self.ip_local.append(self.fileserver_ip) + + if silent: # Restore original functions + if current_parser.exited and self.action == "main": # Argument parsing halted, don't start ZeroNet with main action + self.action = None + current_parser._print_message = original_print_message + current_parser.exit = original_exit + + self.loadTrackersFile() + + # Parse command line arguments + def parseCommandline(self, argv, silent=False): + # Find out if action is specificed on start + action = self.getAction(argv) + if not action: + argv.append("--end") + argv.append("main") + action = "main" + argv = self.moveUnknownToEnd(argv, action) + if silent: + res = self.parser.parse_known_args(argv[1:]) + if res: + self.arguments = res[0] + else: + self.arguments = {} + else: + self.arguments = self.parser.parse_args(argv[1:]) + + # Parse config file + def parseConfig(self, argv): + # Find config file path from parameters + if "--config_file" in argv: + self.config_file = argv[argv.index("--config_file") + 1] + # Load config file + if os.path.isfile(self.config_file): + config = configparser.RawConfigParser(allow_no_value=True, strict=False) + config.read(self.config_file) + for section in config.sections(): + for key, val in config.items(section): + if val == "True": + val = None + if section != "global": # If not global prefix key with section + key = section + "_" + key + + if key == "open_browser": # Prefer config file value over cli argument + while "--%s" % key in argv: + pos = argv.index("--open_browser") + del argv[pos:pos + 2] + + argv_extend = ["--%s" % key] + if val: + for line in val.strip().split("\n"): # Allow multi-line values + argv_extend.append(line) + if "\n" in val: + argv_extend.append("--end") + + argv = argv[:1] + argv_extend + argv[1:] + return argv + + # Return command line value of given argument + def getCmdlineValue(self, key): + if key not in self.argv: + return None + argv_index = self.argv.index(key) + if argv_index == len(self.argv) - 1: # last arg, test not specified + return None + + return self.argv[argv_index + 1] + + # Expose arguments as class attributes + def setAttributes(self): + # Set attributes from arguments + if self.arguments: + args = vars(self.arguments) + for key, val in args.items(): + if type(val) is list: + val = val[:] + if key in ("data_dir", "log_dir", "start_dir", "openssl_bin_file", "openssl_lib_file"): + if val: + val = val.replace("\\", "/") + setattr(self, key, val) + + def loadPlugins(self): + from Plugin import PluginManager + + @PluginManager.acceptPlugins + class ConfigPlugin(object): + def __init__(self, config): + self.argv = config.argv + self.parser = config.parser + self.subparsers = config.subparsers + self.test_parser = config.test_parser + self.getCmdlineValue = config.getCmdlineValue + self.createArguments() + + def createArguments(self): + pass + + ConfigPlugin(self) + + def saveValue(self, key, value): + if not os.path.isfile(self.config_file): + content = "" + else: + content = open(self.config_file).read() + lines = content.splitlines() + + global_line_i = None + key_line_i = None + i = 0 + for line in lines: + if line.strip() == "[global]": + global_line_i = i + if line.startswith(key + " =") or line == key: + key_line_i = i + i += 1 + + if key_line_i and len(lines) > key_line_i + 1: + while True: # Delete previous multiline values + is_value_line = lines[key_line_i + 1].startswith(" ") or lines[key_line_i + 1].startswith("\t") + if not is_value_line: + break + del lines[key_line_i + 1] + + if value is None: # Delete line + if key_line_i: + del lines[key_line_i] + + else: # Add / update + if type(value) is list: + value_lines = [""] + [str(line).replace("\n", "").replace("\r", "") for line in value] + else: + value_lines = [str(value).replace("\n", "").replace("\r", "")] + new_line = "%s = %s" % (key, "\n ".join(value_lines)) + if key_line_i: # Already in the config, change the line + lines[key_line_i] = new_line + elif global_line_i is None: # No global section yet, append to end of file + lines.append("[global]") + lines.append(new_line) + else: # Has global section, append the line after it + lines.insert(global_line_i + 1, new_line) + + open(self.config_file, "w").write("\n".join(lines)) + + def getServerInfo(self): + from Plugin import PluginManager + import main + + info = { + "platform": sys.platform, + "fileserver_ip": self.fileserver_ip, + "fileserver_port": self.fileserver_port, + "ui_ip": self.ui_ip, + "ui_port": self.ui_port, + "version": self.version, + "rev": self.rev, + "language": self.language, + "debug": self.debug, + "plugins": PluginManager.plugin_manager.plugin_names, + + "log_dir": os.path.abspath(self.log_dir), + "data_dir": os.path.abspath(self.data_dir), + "src_dir": os.path.dirname(os.path.abspath(__file__)) + } + + try: + info["ip_external"] = main.file_server.port_opened + info["tor_enabled"] = main.file_server.tor_manager.enabled + info["tor_status"] = main.file_server.tor_manager.status + except Exception: + pass + + return info + + def initConsoleLogger(self): + if self.action == "main": + format = '[%(asctime)s] %(name)s %(message)s' + else: + format = '%(name)s %(message)s' + + if self.console_log_level == "default": + if self.silent: + level = logging.ERROR + elif self.debug: + level = logging.DEBUG + else: + level = logging.INFO + else: + level = logging.getLevelName(self.console_log_level) + + console_logger = logging.StreamHandler() + console_logger.setFormatter(logging.Formatter(format, "%H:%M:%S")) + console_logger.setLevel(level) + logging.getLogger('').addHandler(console_logger) + + def initFileLogger(self): + if self.action == "main": + log_file_path = "%s/debug.log" % self.log_dir + else: + log_file_path = "%s/cmd.log" % self.log_dir + + if self.log_rotate == "off": + file_logger = logging.FileHandler(log_file_path, "w", "utf-8") + else: + when_names = {"weekly": "w", "daily": "d", "hourly": "h"} + file_logger = logging.handlers.TimedRotatingFileHandler( + log_file_path, when=when_names[self.log_rotate], interval=1, backupCount=self.log_rotate_backup_count, + encoding="utf8" + ) + + if os.path.isfile(log_file_path): + file_logger.doRollover() # Always start with empty log file + file_logger.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)-8s %(name)s %(message)s')) + file_logger.setLevel(logging.getLevelName(self.log_level)) + logging.getLogger('').setLevel(logging.getLevelName(self.log_level)) + logging.getLogger('').addHandler(file_logger) + + def initLogging(self, console_logging=None, file_logging=None): + if console_logging == None: + console_logging = self.console_log_level != "off" + + if file_logging == None: + file_logging = self.log_level != "off" + + # Create necessary files and dirs + if not os.path.isdir(self.log_dir): + os.mkdir(self.log_dir) + try: + os.chmod(self.log_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except Exception as err: + print("Can't change permission of %s: %s" % (self.log_dir, err)) + + # Make warning hidden from console + logging.WARNING = 15 # Don't display warnings if not in debug mode + logging.addLevelName(15, "WARNING") + + logging.getLogger('').name = "-" # Remove root prefix + + self.error_logger = ErrorLogHandler() + self.error_logger.setLevel(logging.getLevelName("ERROR")) + logging.getLogger('').addHandler(self.error_logger) + + if console_logging: + self.initConsoleLogger() + if file_logging: + self.initFileLogger() + + +class ErrorLogHandler(logging.StreamHandler): + def __init__(self): + self.lines = [] + return super().__init__() + + def emit(self, record): + self.lines.append([time.time(), record.levelname, self.format(record)]) + + def onNewRecord(self, record): + pass + + +config = Config(sys.argv) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py new file mode 100644 index 00000000..22bcf29c --- /dev/null +++ b/src/Connection/Connection.py @@ -0,0 +1,635 @@ +import socket +import time + +import gevent +try: + from gevent.coros import RLock +except: + from gevent.lock import RLock + +from Config import config +from Debug import Debug +from util import Msgpack +from Crypt import CryptConnection +from util import helper + + +class Connection(object): + __slots__ = ( + "sock", "sock_wrapped", "ip", "port", "cert_pin", "target_onion", "id", "protocol", "type", "server", "unpacker", "unpacker_bytes", "req_id", "ip_type", + "handshake", "crypt", "connected", "event_connected", "closed", "start_time", "handshake_time", "last_recv_time", "is_private_ip", "is_tracker_connection", + "last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent", "cpu_time", "send_lock", + "last_ping_delay", "last_req_time", "last_cmd_sent", "last_cmd_recv", "bad_actions", "sites", "name", "waiting_requests", "waiting_streams" + ) + + def __init__(self, server, ip, port, sock=None, target_onion=None, is_tracker_connection=False): + self.sock = sock + self.cert_pin = None + if "#" in ip: + ip, self.cert_pin = ip.split("#") + self.target_onion = target_onion # Requested onion adress + self.id = server.last_connection_id + server.last_connection_id += 1 + self.protocol = "?" + self.type = "?" + self.ip_type = "?" + self.port = int(port) + self.setIp(ip) + + if helper.isPrivateIp(self.ip) and self.ip not in config.ip_local: + self.is_private_ip = True + else: + self.is_private_ip = False + self.is_tracker_connection = is_tracker_connection + + self.server = server + self.unpacker = None # Stream incoming socket messages here + self.unpacker_bytes = 0 # How many bytes the unpacker received + self.req_id = 0 # Last request id + self.handshake = {} # Handshake info got from peer + self.crypt = None # Connection encryption method + self.sock_wrapped = False # Socket wrapped to encryption + + self.connected = False + self.event_connected = gevent.event.AsyncResult() # Solves on handshake received + self.closed = False + + # Stats + self.start_time = time.time() + self.handshake_time = 0 + self.last_recv_time = 0 + self.last_message_time = 0 + self.last_send_time = 0 + self.last_sent_time = 0 + self.incomplete_buff_recv = 0 + self.bytes_recv = 0 + self.bytes_sent = 0 + self.last_ping_delay = None + self.last_req_time = 0 + self.last_cmd_sent = None + self.last_cmd_recv = None + self.bad_actions = 0 + self.sites = 0 + self.cpu_time = 0.0 + self.send_lock = RLock() + + self.name = None + self.updateName() + + self.waiting_requests = {} # Waiting sent requests + self.waiting_streams = {} # Waiting response file streams + + def setIp(self, ip): + self.ip = ip + self.ip_type = helper.getIpType(ip) + self.updateName() + + def createSocket(self): + if helper.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"): + # Create IPv6 connection as IPv4 when using proxy + return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def updateName(self): + self.name = "Conn#%2s %-12s [%s]" % (self.id, self.ip, self.protocol) + + def __str__(self): + return self.name + + def __repr__(self): + return "<%s>" % self.__str__() + + def log(self, text): + self.server.log.debug("%s > %s" % (self.name, text)) + + def getValidSites(self): + return [key for key, val in self.server.tor_manager.site_onions.items() if val == self.target_onion] + + def badAction(self, weight=1): + self.bad_actions += weight + if self.bad_actions > 40: + self.close("Too many bad actions") + elif self.bad_actions > 20: + time.sleep(5) + + def goodAction(self): + self.bad_actions = 0 + + # Open connection to peer and wait for handshake + def connect(self): + self.type = "out" + if self.ip_type == "onion": + if not self.server.tor_manager or not self.server.tor_manager.enabled: + raise Exception("Can't connect to onion addresses, no Tor controller present") + self.sock = self.server.tor_manager.createSocket(self.ip, self.port) + elif config.tor == "always" and helper.isPrivateIp(self.ip) and self.ip not in config.ip_local: + raise Exception("Can't connect to local IPs in Tor: always mode") + elif config.trackers_proxy != "disable" and config.tor != "always" and self.is_tracker_connection: + if config.trackers_proxy == "tor": + self.sock = self.server.tor_manager.createSocket(self.ip, self.port) + else: + import socks + self.sock = socks.socksocket() + proxy_ip, proxy_port = config.trackers_proxy.split(":") + self.sock.set_proxy(socks.PROXY_TYPE_SOCKS5, proxy_ip, int(proxy_port)) + else: + self.sock = self.createSocket() + + if "TCP_NODELAY" in dir(socket): + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + timeout_before = self.sock.gettimeout() + self.sock.settimeout(30) + if self.ip_type == "ipv6" and not hasattr(self.sock, "proxy"): + sock_address = (self.ip, self.port, 1, 1) + else: + sock_address = (self.ip, self.port) + + self.sock.connect(sock_address) + + # Implicit SSL + should_encrypt = not self.ip_type == "onion" and self.ip not in self.server.broken_ssl_ips and self.ip not in config.ip_local + if self.cert_pin: + self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", cert_pin=self.cert_pin) + self.sock.do_handshake() + self.crypt = "tls-rsa" + self.sock_wrapped = True + elif should_encrypt and "tls-rsa" in CryptConnection.manager.crypt_supported: + try: + self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa") + self.sock.do_handshake() + self.crypt = "tls-rsa" + self.sock_wrapped = True + except Exception as err: + if not config.force_encryption: + self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err))) + self.server.broken_ssl_ips[self.ip] = True + self.sock.close() + self.crypt = None + self.sock = self.createSocket() + self.sock.settimeout(30) + self.sock.connect(sock_address) + + # Detect protocol + self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()}) + event_connected = self.event_connected + gevent.spawn(self.messageLoop) + connect_res = event_connected.get() # Wait for handshake + self.sock.settimeout(timeout_before) + return connect_res + + # Handle incoming connection + def handleIncomingConnection(self, sock): + self.log("Incoming connection...") + + if "TCP_NODELAY" in dir(socket): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + self.type = "in" + if self.ip not in config.ip_local: # Clearnet: Check implicit SSL + try: + first_byte = sock.recv(1, gevent.socket.MSG_PEEK) + if first_byte == b"\x16": + self.log("Crypt in connection using implicit SSL") + self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", True) + self.sock_wrapped = True + self.crypt = "tls-rsa" + except Exception as err: + self.log("Socket peek error: %s" % Debug.formatException(err)) + self.messageLoop() + + def getMsgpackUnpacker(self): + if self.handshake and self.handshake.get("use_bin_type"): + return Msgpack.getUnpacker(fallback=True, decode=False) + else: # Backward compatibility for <0.7.0 + return Msgpack.getUnpacker(fallback=True, decode=True) + + # Message loop for connection + def messageLoop(self): + if not self.sock: + self.log("Socket error: No socket found") + return False + self.protocol = "v2" + self.updateName() + self.connected = True + buff_len = 0 + req_len = 0 + self.unpacker_bytes = 0 + + try: + while not self.closed: + buff = self.sock.recv(64 * 1024) + if not buff: + break # Connection closed + buff_len = len(buff) + + # Statistics + self.last_recv_time = time.time() + self.incomplete_buff_recv += 1 + self.bytes_recv += buff_len + self.server.bytes_recv += buff_len + req_len += buff_len + + if not self.unpacker: + self.unpacker = self.getMsgpackUnpacker() + self.unpacker_bytes = 0 + + self.unpacker.feed(buff) + self.unpacker_bytes += buff_len + + while True: + try: + message = next(self.unpacker) + except StopIteration: + break + if not type(message) is dict: + if config.debug_socket: + self.log("Invalid message type: %s, content: %r, buffer: %r" % (type(message), message, buff[0:16])) + raise Exception("Invalid message type: %s" % type(message)) + + # Stats + self.incomplete_buff_recv = 0 + stat_key = message.get("cmd", "unknown") + if stat_key == "response" and "to" in message: + cmd_sent = self.waiting_requests.get(message["to"], {"cmd": "unknown"})["cmd"] + stat_key = "response: %s" % cmd_sent + if stat_key == "update": + stat_key = "update: %s" % message["params"]["site"] + self.server.stat_recv[stat_key]["bytes"] += req_len + self.server.stat_recv[stat_key]["num"] += 1 + if "stream_bytes" in message: + self.server.stat_recv[stat_key]["bytes"] += message["stream_bytes"] + req_len = 0 + + # Handle message + if "stream_bytes" in message: + buff_left = self.handleStream(message, buff) + self.unpacker = self.getMsgpackUnpacker() + self.unpacker.feed(buff_left) + self.unpacker_bytes = len(buff_left) + if config.debug_socket: + self.log("Start new unpacker with buff_left: %r" % buff_left) + else: + self.handleMessage(message) + + message = None + except Exception as err: + if not self.closed: + self.log("Socket error: %s" % Debug.formatException(err)) + self.server.stat_recv["error: %s" % err]["bytes"] += req_len + self.server.stat_recv["error: %s" % err]["num"] += 1 + self.close("MessageLoop ended (closed: %s)" % self.closed) # MessageLoop ended, close connection + + def getUnpackerUnprocessedBytesNum(self): + if "tell" in dir(self.unpacker): + bytes_num = self.unpacker_bytes - self.unpacker.tell() + else: + bytes_num = self.unpacker._fb_buf_n - self.unpacker._fb_buf_o + return bytes_num + + # Stream socket directly to a file + def handleStream(self, message, buff): + stream_bytes_left = message["stream_bytes"] + file = self.waiting_streams[message["to"]] + + unprocessed_bytes_num = self.getUnpackerUnprocessedBytesNum() + + if unprocessed_bytes_num: # Found stream bytes in unpacker + unpacker_stream_bytes = min(unprocessed_bytes_num, stream_bytes_left) + buff_stream_start = len(buff) - unprocessed_bytes_num + file.write(buff[buff_stream_start:buff_stream_start + unpacker_stream_bytes]) + stream_bytes_left -= unpacker_stream_bytes + else: + unpacker_stream_bytes = 0 + + if config.debug_socket: + self.log( + "Starting stream %s: %s bytes (%s from unpacker, buff size: %s, unprocessed: %s)" % + (message["to"], message["stream_bytes"], unpacker_stream_bytes, len(buff), unprocessed_bytes_num) + ) + + try: + while 1: + if stream_bytes_left <= 0: + break + stream_buff = self.sock.recv(min(64 * 1024, stream_bytes_left)) + if not stream_buff: + break + buff_len = len(stream_buff) + stream_bytes_left -= buff_len + file.write(stream_buff) + + # Statistics + self.last_recv_time = time.time() + self.incomplete_buff_recv += 1 + self.bytes_recv += buff_len + self.server.bytes_recv += buff_len + except Exception as err: + self.log("Stream read error: %s" % Debug.formatException(err)) + + if config.debug_socket: + self.log("End stream %s, file pos: %s" % (message["to"], file.tell())) + + self.incomplete_buff_recv = 0 + self.waiting_requests[message["to"]]["evt"].set(message) # Set the response to event + del self.waiting_streams[message["to"]] + del self.waiting_requests[message["to"]] + + if unpacker_stream_bytes: + return buff[buff_stream_start + unpacker_stream_bytes:] + else: + return b"" + + # My handshake info + def getHandshakeInfo(self): + # No TLS for onion connections + if self.ip_type == "onion": + crypt_supported = [] + elif self.ip in self.server.broken_ssl_ips: + crypt_supported = [] + else: + crypt_supported = CryptConnection.manager.crypt_supported + # No peer id for onion connections + if self.ip_type == "onion" or self.ip in config.ip_local: + peer_id = "" + else: + peer_id = self.server.peer_id + # Setup peer lock from requested onion address + if self.handshake and self.handshake.get("target_ip", "").endswith(".onion") and self.server.tor_manager.start_onions: + self.target_onion = self.handshake.get("target_ip").replace(".onion", "") # My onion address + if not self.server.tor_manager.site_onions.values(): + self.server.log.warning("Unknown target onion address: %s" % self.target_onion) + + handshake = { + "version": config.version, + "protocol": "v2", + "use_bin_type": True, + "peer_id": peer_id, + "fileserver_port": self.server.port, + "port_opened": self.server.port_opened.get(self.ip_type, None), + "target_ip": self.ip, + "rev": config.rev, + "crypt_supported": crypt_supported, + "crypt": self.crypt, + "time": int(time.time()) + } + if self.target_onion: + handshake["onion"] = self.target_onion + elif self.ip_type == "onion": + handshake["onion"] = self.server.tor_manager.getOnion("global") + + if self.is_tracker_connection: + handshake["tracker_connection"] = True + + if config.debug_socket: + self.log("My Handshake: %s" % handshake) + + return handshake + + def setHandshake(self, handshake): + if config.debug_socket: + self.log("Remote Handshake: %s" % handshake) + + if handshake.get("peer_id") == self.server.peer_id and not handshake.get("tracker_connection") and not self.is_tracker_connection: + self.close("Same peer id, can't connect to myself") + self.server.peer_blacklist.append((handshake["target_ip"], handshake["fileserver_port"])) + return False + + self.handshake = handshake + if handshake.get("port_opened", None) is False and "onion" not in handshake and not self.is_private_ip: # Not connectable + self.port = 0 + else: + self.port = int(handshake["fileserver_port"]) # Set peer fileserver port + + if handshake.get("use_bin_type") and self.unpacker: + unprocessed_bytes_num = self.getUnpackerUnprocessedBytesNum() + self.log("Changing unpacker to bin type (unprocessed bytes: %s)" % unprocessed_bytes_num) + unprocessed_bytes = self.unpacker.read_bytes(unprocessed_bytes_num) + self.unpacker = self.getMsgpackUnpacker() # Create new unpacker for different msgpack type + self.unpacker_bytes = 0 + if unprocessed_bytes: + self.unpacker.feed(unprocessed_bytes) + + # Check if we can encrypt the connection + if handshake.get("crypt_supported") and self.ip not in self.server.broken_ssl_ips: + if type(handshake["crypt_supported"][0]) is bytes: + handshake["crypt_supported"] = [item.decode() for item in handshake["crypt_supported"]] # Backward compatibility + + if self.ip_type == "onion" or self.ip in config.ip_local: + crypt = None + elif handshake.get("crypt"): # Recommended crypt by server + crypt = handshake["crypt"] + else: # Select the best supported on both sides + crypt = CryptConnection.manager.selectCrypt(handshake["crypt_supported"]) + + if crypt: + self.crypt = crypt + + if self.type == "in" and handshake.get("onion") and not self.ip_type == "onion": # Set incoming connection's onion address + if self.server.ips.get(self.ip) == self: + del self.server.ips[self.ip] + self.setIp(handshake["onion"] + ".onion") + self.log("Changing ip to %s" % self.ip) + self.server.ips[self.ip] = self + self.updateName() + + self.event_connected.set(True) # Mark handshake as done + self.event_connected = None + self.handshake_time = time.time() + + # Handle incoming message + def handleMessage(self, message): + cmd = message["cmd"] + + self.last_message_time = time.time() + self.last_cmd_recv = cmd + if cmd == "response": # New style response + if message["to"] in self.waiting_requests: + if self.last_send_time and len(self.waiting_requests) == 1: + ping = time.time() - self.last_send_time + self.last_ping_delay = ping + self.waiting_requests[message["to"]]["evt"].set(message) # Set the response to event + del self.waiting_requests[message["to"]] + elif message["to"] == 0: # Other peers handshake + ping = time.time() - self.start_time + if config.debug_socket: + self.log("Handshake response: %s, ping: %s" % (message, ping)) + self.last_ping_delay = ping + # Server switched to crypt, lets do it also if not crypted already + if message.get("crypt") and not self.sock_wrapped: + self.crypt = message["crypt"] + server = (self.type == "in") + self.log("Crypt out connection using: %s (server side: %s, ping: %.3fs)..." % (self.crypt, server, ping)) + self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin) + self.sock.do_handshake() + self.sock_wrapped = True + + if not self.sock_wrapped and self.cert_pin: + self.close("Crypt connection error: Socket not encrypted, but certificate pin present") + return + + self.setHandshake(message) + else: + self.log("Unknown response: %s" % message) + elif cmd: + self.server.num_recv += 1 + if cmd == "handshake": + self.handleHandshake(message) + else: + self.server.handleRequest(self, message) + + # Incoming handshake set request + def handleHandshake(self, message): + self.setHandshake(message["params"]) + data = self.getHandshakeInfo() + data["cmd"] = "response" + data["to"] = message["req_id"] + self.send(data) # Send response to handshake + # Sent crypt request to client + if self.crypt and not self.sock_wrapped: + server = (self.type == "in") + self.log("Crypt in connection using: %s (server side: %s)..." % (self.crypt, server)) + try: + self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin) + self.sock_wrapped = True + except Exception as err: + if not config.force_encryption: + self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err))) + self.server.broken_ssl_ips[self.ip] = True + self.close("Broken ssl") + + if not self.sock_wrapped and self.cert_pin: + self.close("Crypt connection error: Socket not encrypted, but certificate pin present") + + # Send data to connection + def send(self, message, streaming=False): + self.last_send_time = time.time() + if config.debug_socket: + self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % ( + message.get("cmd"), message.get("to"), streaming, + message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), + message.get("req_id")) + ) + + if not self.sock: + self.log("Send error: missing socket") + return False + + if not self.connected and message.get("cmd") != "handshake": + self.log("Wait for handshake before send request") + self.event_connected.get() + + try: + stat_key = message.get("cmd", "unknown") + if stat_key == "response": + stat_key = "response: %s" % self.last_cmd_recv + else: + self.server.num_sent += 1 + + self.server.stat_sent[stat_key]["num"] += 1 + if streaming: + with self.send_lock: + bytes_sent = Msgpack.stream(message, self.sock.sendall) + self.bytes_sent += bytes_sent + self.server.bytes_sent += bytes_sent + self.server.stat_sent[stat_key]["bytes"] += bytes_sent + message = None + else: + data = Msgpack.pack(message) + self.bytes_sent += len(data) + self.server.bytes_sent += len(data) + self.server.stat_sent[stat_key]["bytes"] += len(data) + message = None + with self.send_lock: + self.sock.sendall(data) + except Exception as err: + self.close("Send error: %s (cmd: %s)" % (err, stat_key)) + return False + self.last_sent_time = time.time() + return True + + # Stream file to connection without msgpacking + def sendRawfile(self, file, read_bytes): + buff = 64 * 1024 + bytes_left = read_bytes + bytes_sent = 0 + while True: + self.last_send_time = time.time() + data = file.read(min(bytes_left, buff)) + bytes_sent += len(data) + with self.send_lock: + self.sock.sendall(data) + bytes_left -= buff + if bytes_left <= 0: + break + self.bytes_sent += bytes_sent + self.server.bytes_sent += bytes_sent + self.server.stat_sent["raw_file"]["num"] += 1 + self.server.stat_sent["raw_file"]["bytes"] += bytes_sent + return True + + # Create and send a request to peer + def request(self, cmd, params={}, stream_to=None): + # Last command sent more than 10 sec ago, timeout + if self.waiting_requests and self.protocol == "v2" and time.time() - max(self.last_req_time, self.last_recv_time) > 10: + self.close("Request %s timeout: %.3fs" % (self.last_cmd_sent, time.time() - self.last_send_time)) + return False + + self.last_req_time = time.time() + self.last_cmd_sent = cmd + self.req_id += 1 + data = {"cmd": cmd, "req_id": self.req_id, "params": params} + event = gevent.event.AsyncResult() # Create new event for response + self.waiting_requests[self.req_id] = {"evt": event, "cmd": cmd} + if stream_to: + self.waiting_streams[self.req_id] = stream_to + self.send(data) # Send request + res = event.get() # Wait until event solves + return res + + def ping(self): + s = time.time() + response = None + with gevent.Timeout(10.0, False): + try: + response = self.request("ping") + except Exception as err: + self.log("Ping error: %s" % Debug.formatException(err)) + if response and "body" in response and response["body"] == b"Pong!": + self.last_ping_delay = time.time() - s + return True + else: + return False + + # Close connection + def close(self, reason="Unknown"): + if self.closed: + return False # Already closed + self.closed = True + self.connected = False + if self.event_connected: + self.event_connected.set(False) + + self.log( + "Closing connection: %s, waiting_requests: %s, sites: %s, buff: %s..." % + (reason, len(self.waiting_requests), self.sites, self.incomplete_buff_recv) + ) + for request in self.waiting_requests.values(): # Mark pending requests failed + request["evt"].set(False) + self.waiting_requests = {} + self.waiting_streams = {} + self.sites = 0 + self.server.removeConnection(self) # Remove connection from server registry + try: + if self.sock: + self.sock.shutdown(gevent.socket.SHUT_WR) + self.sock.close() + except Exception as err: + if config.debug_socket: + self.log("Close error: %s" % err) + + # Little cleanup + self.sock = None + self.unpacker = None + self.event_connected = None diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py new file mode 100644 index 00000000..c9048398 --- /dev/null +++ b/src/Connection/ConnectionServer.py @@ -0,0 +1,386 @@ +import logging +import time +import sys +import socket +from collections import defaultdict + +import gevent +import msgpack +from gevent.server import StreamServer +from gevent.pool import Pool + +import util +from util import helper +from Debug import Debug +from .Connection import Connection +from Config import config +from Crypt import CryptConnection +from Crypt import CryptHash +from Tor import TorManager +from Site import SiteManager + + +class ConnectionServer(object): + def __init__(self, ip=None, port=None, request_handler=None): + if not ip: + if config.fileserver_ip_type == "ipv6": + ip = "::1" + else: + ip = "127.0.0.1" + port = 15441 + self.ip = ip + self.port = port + self.last_connection_id = 0 # Connection id incrementer + self.last_connection_id_current_version = 0 # Connection id incrementer for current client version + self.last_connection_id_supported_version = 0 # Connection id incrementer for last supported version + self.log = logging.getLogger("ConnServer") + self.port_opened = {} + self.peer_blacklist = SiteManager.peer_blacklist + + self.tor_manager = TorManager(self.ip, self.port) + self.connections = [] # Connections + self.whitelist = config.ip_local # No flood protection on this ips + self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood + self.broken_ssl_ips = {} # Peerids of broken ssl connections + self.ips = {} # Connection by ip + self.has_internet = True # Internet outage detection + + self.stream_server = None + self.stream_server_proxy = None + self.running = False + self.stopping = False + self.thread_checker = None + + self.stat_recv = defaultdict(lambda: defaultdict(int)) + self.stat_sent = defaultdict(lambda: defaultdict(int)) + self.bytes_recv = 0 + self.bytes_sent = 0 + self.num_recv = 0 + self.num_sent = 0 + + self.num_incoming = 0 + self.num_outgoing = 0 + self.had_external_incoming = False + + self.timecorrection = 0.0 + self.pool = Pool(500) # do not accept more than 500 connections + + # Bittorrent style peerid + self.peer_id = "-UT3530-%s" % CryptHash.random(12, "base64") + + # Check msgpack version + if msgpack.version[0] == 0 and msgpack.version[1] < 4: + self.log.error( + "Error: Unsupported msgpack version: %s (<0.4.0), please run `sudo apt-get install python-pip; sudo pip install msgpack --upgrade`" % + str(msgpack.version) + ) + sys.exit(0) + + if request_handler: + self.handleRequest = request_handler + + def start(self, check_connections=True): + if self.stopping: + return False + self.running = True + if check_connections: + self.thread_checker = gevent.spawn(self.checkConnections) + CryptConnection.manager.loadCerts() + if config.tor != "disable": + self.tor_manager.start() + if not self.port: + self.log.info("No port found, not binding") + return False + + self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % ( + self.ip, self.port, ".".join(map(str, msgpack.version)), + CryptConnection.manager.crypt_supported + )) + try: + self.stream_server = StreamServer( + (self.ip, self.port), self.handleIncomingConnection, spawn=self.pool, backlog=100 + ) + except Exception as err: + self.log.info("StreamServer create error: %s" % Debug.formatException(err)) + + def listen(self): + if not self.running: + return None + + if self.stream_server_proxy: + gevent.spawn(self.listenProxy) + try: + self.stream_server.serve_forever() + except Exception as err: + self.log.info("StreamServer listen error: %s" % err) + return False + self.log.debug("Stopped.") + + def stop(self): + self.log.debug("Stopping %s" % self.stream_server) + self.stopping = True + self.running = False + if self.thread_checker: + gevent.kill(self.thread_checker) + if self.stream_server: + self.stream_server.stop() + + def closeConnections(self): + self.log.debug("Closing all connection: %s" % len(self.connections)) + for connection in self.connections[:]: + connection.close("Close all connections") + + def handleIncomingConnection(self, sock, addr): + if config.offline: + sock.close() + return False + + ip, port = addr[0:2] + ip = ip.lower() + if ip.startswith("::ffff:"): # IPv6 to IPv4 mapping + ip = ip.replace("::ffff:", "", 1) + self.num_incoming += 1 + + if not self.had_external_incoming and not helper.isPrivateIp(ip): + self.had_external_incoming = True + + # Connection flood protection + if ip in self.ip_incoming and ip not in self.whitelist: + self.ip_incoming[ip] += 1 + if self.ip_incoming[ip] > 6: # Allow 6 in 1 minute from same ip + self.log.debug("Connection flood detected from %s" % ip) + time.sleep(30) + sock.close() + return False + else: + self.ip_incoming[ip] = 1 + + connection = Connection(self, ip, port, sock) + self.connections.append(connection) + rev = connection.handshake.get("rev", 0) + if rev >= 4560: + self.last_connection_id_supported_version += 1 + if rev == config.rev: + self.last_connection_id_current_version += 1 + if ip not in config.ip_local: + self.ips[ip] = connection + connection.handleIncomingConnection(sock) + + def handleMessage(self, *args, **kwargs): + pass + + def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None, is_tracker_connection=False): + ip_type = helper.getIpType(ip) + has_per_site_onion = (ip.endswith(".onion") or self.port_opened.get(ip_type, None) == False) and self.tor_manager.start_onions and site + if has_per_site_onion: # Site-unique connection for Tor + if ip.endswith(".onion"): + site_onion = self.tor_manager.getOnion(site.address) + else: + site_onion = self.tor_manager.getOnion("global") + key = ip + site_onion + else: + key = ip + + # Find connection by ip + if key in self.ips: + connection = self.ips[key] + if not peer_id or connection.handshake.get("peer_id") == peer_id: # Filter by peer_id + if not connection.connected and create: + succ = connection.event_connected.get() # Wait for connection + if not succ: + raise Exception("Connection event return error") + return connection + + # Recover from connection pool + for connection in self.connections: + if connection.ip == ip: + if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match + continue + if ip.endswith(".onion") and self.tor_manager.start_onions and ip.replace(".onion", "") != connection.target_onion: + # For different site + continue + if not connection.connected and create: + succ = connection.event_connected.get() # Wait for connection + if not succ: + raise Exception("Connection event return error") + return connection + + # No connection found + if create and not config.offline: # Allow to create new connection if not found + if port == 0: + raise Exception("This peer is not connectable") + + if (ip, port) in self.peer_blacklist and not is_tracker_connection: + raise Exception("This peer is blacklisted") + + try: + if has_per_site_onion: # Lock connection to site + connection = Connection(self, ip, port, target_onion=site_onion, is_tracker_connection=is_tracker_connection) + else: + connection = Connection(self, ip, port, is_tracker_connection=is_tracker_connection) + self.num_outgoing += 1 + self.ips[key] = connection + self.connections.append(connection) + connection.log("Connecting... (site: %s)" % site) + succ = connection.connect() + if not succ: + connection.close("Connection event return error") + raise Exception("Connection event return error") + else: + rev = connection.handshake.get("rev", 0) + if rev >= 4560: + self.last_connection_id_supported_version += 1 + if rev == config.rev: + self.last_connection_id_current_version += 1 + + except Exception as err: + connection.close("%s Connect error: %s" % (ip, Debug.formatException(err))) + raise err + + if len(self.connections) > config.global_connected_limit: + gevent.spawn(self.checkMaxConnections) + + return connection + else: + return None + + def removeConnection(self, connection): + # Delete if same as in registry + if self.ips.get(connection.ip) == connection: + del self.ips[connection.ip] + # Site locked connection + if connection.target_onion: + if self.ips.get(connection.ip + connection.target_onion) == connection: + del self.ips[connection.ip + connection.target_onion] + # Cert pinned connection + if connection.cert_pin and self.ips.get(connection.ip + "#" + connection.cert_pin) == connection: + del self.ips[connection.ip + "#" + connection.cert_pin] + + if connection in self.connections: + self.connections.remove(connection) + + def checkConnections(self): + run_i = 0 + time.sleep(15) + while self.running: + run_i += 1 + self.ip_incoming = {} # Reset connected ips counter + last_message_time = 0 + s = time.time() + for connection in self.connections[:]: # Make a copy + if connection.ip.endswith(".onion") or config.tor == "always": + timeout_multipler = 2 + else: + timeout_multipler = 1 + + idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time) + if connection.last_message_time > last_message_time and not connection.is_private_ip: + # Message from local IPs does not means internet connection + last_message_time = connection.last_message_time + + if connection.unpacker and idle > 30: + # Delete the unpacker if not needed + del connection.unpacker + connection.unpacker = None + + elif connection.last_cmd_sent == "announce" and idle > 20: # Bootstrapper connection close after 20 sec + connection.close("[Cleanup] Tracker connection, idle: %.3fs" % idle) + + if idle > 60 * 60: + # Wake up after 1h + connection.close("[Cleanup] After wakeup, idle: %.3fs" % idle) + + elif idle > 20 * 60 and connection.last_send_time < time.time() - 10: + # Idle more than 20 min and we have not sent request in last 10 sec + if not connection.ping(): + connection.close("[Cleanup] Ping timeout") + + elif idle > 10 * timeout_multipler and connection.incomplete_buff_recv > 0: + # Incomplete data with more than 10 sec idle + connection.close("[Cleanup] Connection buff stalled") + + elif idle > 10 * timeout_multipler and connection.protocol == "?": # No connection after 10 sec + connection.close( + "[Cleanup] Connect timeout: %.3fs" % idle + ) + + elif idle > 10 * timeout_multipler and connection.waiting_requests and time.time() - connection.last_send_time > 10 * timeout_multipler: + # Sent command and no response in 10 sec + connection.close( + "[Cleanup] Command %s timeout: %.3fs" % (connection.last_cmd_sent, time.time() - connection.last_send_time) + ) + + elif idle < 60 and connection.bad_actions > 40: + connection.close( + "[Cleanup] Too many bad actions: %s" % connection.bad_actions + ) + + elif idle > 5 * 60 and connection.sites == 0: + connection.close( + "[Cleanup] No site for connection" + ) + + elif run_i % 90 == 0: + # Reset bad action counter every 30 min + connection.bad_actions = 0 + + # Internet outage detection + if time.time() - last_message_time > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)): + # Offline: Last message more than 60-600sec depending on connection number + if self.has_internet and last_message_time: + self.has_internet = False + self.onInternetOffline() + else: + # Online + if not self.has_internet: + self.has_internet = True + self.onInternetOnline() + + self.timecorrection = self.getTimecorrection() + + if time.time() - s > 0.01: + self.log.debug("Connection cleanup in %.3fs" % (time.time() - s)) + + time.sleep(15) + self.log.debug("Checkconnections ended") + + @util.Noparallel(blocking=False) + def checkMaxConnections(self): + if len(self.connections) < config.global_connected_limit: + return 0 + + s = time.time() + num_connected_before = len(self.connections) + self.connections.sort(key=lambda connection: connection.sites) + num_closed = 0 + for connection in self.connections: + idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time) + if idle > 60: + connection.close("Connection limit reached") + num_closed += 1 + if num_closed > config.global_connected_limit * 0.1: + break + + self.log.debug("Closed %s connections of %s after reached limit %s in %.3fs" % ( + num_closed, num_connected_before, config.global_connected_limit, time.time() - s + )) + return num_closed + + def onInternetOnline(self): + self.log.info("Internet online") + + def onInternetOffline(self): + self.had_external_incoming = False + self.log.info("Internet offline") + + def getTimecorrection(self): + corrections = sorted([ + connection.handshake.get("time") - connection.handshake_time + connection.last_ping_delay + for connection in self.connections + if connection.handshake.get("time") and connection.last_ping_delay + ]) + if len(corrections) < 9: + return 0.0 + mid = int(len(corrections) / 2 - 1) + median = (corrections[mid - 1] + corrections[mid] + corrections[mid + 1]) / 3 + return median diff --git a/src/Connection/__init__.py b/src/Connection/__init__.py new file mode 100644 index 00000000..d419a3f0 --- /dev/null +++ b/src/Connection/__init__.py @@ -0,0 +1,2 @@ +from .ConnectionServer import ConnectionServer +from .Connection import Connection diff --git a/src/Content/ContentDb.py b/src/Content/ContentDb.py new file mode 100644 index 00000000..f284581e --- /dev/null +++ b/src/Content/ContentDb.py @@ -0,0 +1,162 @@ +import os + +from Db.Db import Db, DbTableError +from Config import config +from Plugin import PluginManager +from Debug import Debug + + +@PluginManager.acceptPlugins +class ContentDb(Db): + def __init__(self, path): + Db.__init__(self, {"db_name": "ContentDb", "tables": {}}, path) + self.foreign_keys = True + + def init(self): + try: + self.schema = self.getSchema() + try: + self.checkTables() + except DbTableError: + pass + self.log.debug("Checking foreign keys...") + foreign_key_error = self.execute("PRAGMA foreign_key_check").fetchone() + if foreign_key_error: + raise Exception("Database foreign key error: %s" % foreign_key_error) + except Exception as err: + self.log.error("Error loading content.db: %s, rebuilding..." % Debug.formatException(err)) + self.close() + os.unlink(self.db_path) # Remove and try again + Db.__init__(self, {"db_name": "ContentDb", "tables": {}}, self.db_path) + self.foreign_keys = True + self.schema = self.getSchema() + try: + self.checkTables() + except DbTableError: + pass + self.site_ids = {} + self.sites = {} + + def getSchema(self): + schema = {} + schema["db_name"] = "ContentDb" + schema["version"] = 3 + schema["tables"] = {} + + if not self.getTableVersion("site"): + self.log.debug("Migrating from table version-less content.db") + version = int(self.execute("PRAGMA user_version").fetchone()[0]) + if version > 0: + self.checkTables() + self.execute("INSERT INTO keyvalue ?", {"json_id": 0, "key": "table.site.version", "value": 1}) + self.execute("INSERT INTO keyvalue ?", {"json_id": 0, "key": "table.content.version", "value": 1}) + + schema["tables"]["site"] = { + "cols": [ + ["site_id", "INTEGER PRIMARY KEY ASC NOT NULL UNIQUE"], + ["address", "TEXT NOT NULL"] + ], + "indexes": [ + "CREATE UNIQUE INDEX site_address ON site (address)" + ], + "schema_changed": 1 + } + + schema["tables"]["content"] = { + "cols": [ + ["content_id", "INTEGER PRIMARY KEY UNIQUE NOT NULL"], + ["site_id", "INTEGER REFERENCES site (site_id) ON DELETE CASCADE"], + ["inner_path", "TEXT"], + ["size", "INTEGER"], + ["size_files", "INTEGER"], + ["size_files_optional", "INTEGER"], + ["modified", "INTEGER"] + ], + "indexes": [ + "CREATE UNIQUE INDEX content_key ON content (site_id, inner_path)", + "CREATE INDEX content_modified ON content (site_id, modified)" + ], + "schema_changed": 1 + } + + return schema + + def initSite(self, site): + self.sites[site.address] = site + + def needSite(self, site): + if site.address not in self.site_ids: + self.execute("INSERT OR IGNORE INTO site ?", {"address": site.address}) + self.site_ids = {} + for row in self.execute("SELECT * FROM site"): + self.site_ids[row["address"]] = row["site_id"] + return self.site_ids[site.address] + + def deleteSite(self, site): + site_id = self.site_ids.get(site.address, 0) + if site_id: + self.execute("DELETE FROM site WHERE site_id = :site_id", {"site_id": site_id}) + del self.site_ids[site.address] + del self.sites[site.address] + + def setContent(self, site, inner_path, content, size=0): + self.insertOrUpdate("content", { + "size": size, + "size_files": sum([val["size"] for key, val in content.get("files", {}).items()]), + "size_files_optional": sum([val["size"] for key, val in content.get("files_optional", {}).items()]), + "modified": int(content.get("modified", 0)) + }, { + "site_id": self.site_ids.get(site.address, 0), + "inner_path": inner_path + }) + + def deleteContent(self, site, inner_path): + self.execute("DELETE FROM content WHERE ?", {"site_id": self.site_ids.get(site.address, 0), "inner_path": inner_path}) + + def loadDbDict(self, site): + res = self.execute( + "SELECT GROUP_CONCAT(inner_path, '|') AS inner_paths FROM content WHERE ?", + {"site_id": self.site_ids.get(site.address, 0)} + ) + row = res.fetchone() + if row and row["inner_paths"]: + inner_paths = row["inner_paths"].split("|") + return dict.fromkeys(inner_paths, False) + else: + return {} + + def getTotalSize(self, site, ignore=None): + params = {"site_id": self.site_ids.get(site.address, 0)} + if ignore: + params["not__inner_path"] = ignore + res = self.execute("SELECT SUM(size) + SUM(size_files) AS size, SUM(size_files_optional) AS size_optional FROM content WHERE ?", params) + row = dict(res.fetchone()) + + if not row["size"]: + row["size"] = 0 + if not row["size_optional"]: + row["size_optional"] = 0 + + return row["size"], row["size_optional"] + + def listModified(self, site, after=None, before=None): + params = {"site_id": self.site_ids.get(site.address, 0)} + if after: + params["modified>"] = after + if before: + params["modified<"] = before + res = self.execute("SELECT inner_path, modified FROM content WHERE ?", params) + return {row["inner_path"]: row["modified"] for row in res} + +content_dbs = {} + + +def getContentDb(path=None): + if not path: + path = "%s/content.db" % config.data_dir + if path not in content_dbs: + content_dbs[path] = ContentDb(path) + content_dbs[path].init() + return content_dbs[path] + +getContentDb() # Pre-connect to default one diff --git a/src/Content/ContentDbDict.py b/src/Content/ContentDbDict.py new file mode 100644 index 00000000..01df0427 --- /dev/null +++ b/src/Content/ContentDbDict.py @@ -0,0 +1,155 @@ +import time +import os + +from . import ContentDb +from Debug import Debug +from Config import config + + +class ContentDbDict(dict): + def __init__(self, site, *args, **kwargs): + s = time.time() + self.site = site + self.cached_keys = [] + self.log = self.site.log + self.db = ContentDb.getContentDb() + self.db_id = self.db.needSite(site) + self.num_loaded = 0 + super(ContentDbDict, self).__init__(self.db.loadDbDict(site)) # Load keys from database + self.log.debug("ContentDb init: %.3fs, found files: %s, sites: %s" % (time.time() - s, len(self), len(self.db.site_ids))) + + def loadItem(self, key): + try: + self.num_loaded += 1 + if self.num_loaded % 100 == 0: + if config.verbose: + self.log.debug("Loaded json: %s (latest: %s) called by: %s" % (self.num_loaded, key, Debug.formatStack())) + else: + self.log.debug("Loaded json: %s (latest: %s)" % (self.num_loaded, key)) + content = self.site.storage.loadJson(key) + dict.__setitem__(self, key, content) + except IOError: + if dict.get(self, key): + self.__delitem__(key) # File not exists anymore + raise KeyError(key) + + self.addCachedKey(key) + self.checkLimit() + + return content + + def getItemSize(self, key): + return self.site.storage.getSize(key) + + # Only keep last 10 accessed json in memory + def checkLimit(self): + if len(self.cached_keys) > 10: + key_deleted = self.cached_keys.pop(0) + dict.__setitem__(self, key_deleted, False) + + def addCachedKey(self, key): + if key not in self.cached_keys and key != "content.json" and len(key) > 40: # Always keep keys smaller than 40 char + self.cached_keys.append(key) + + def __getitem__(self, key): + val = dict.get(self, key) + if val: # Already loaded + return val + elif val is None: # Unknown key + raise KeyError(key) + elif val is False: # Loaded before, but purged from cache + return self.loadItem(key) + + def __setitem__(self, key, val): + self.addCachedKey(key) + self.checkLimit() + size = self.getItemSize(key) + self.db.setContent(self.site, key, val, size) + dict.__setitem__(self, key, val) + + def __delitem__(self, key): + self.db.deleteContent(self.site, key) + dict.__delitem__(self, key) + try: + self.cached_keys.remove(key) + except ValueError: + pass + + def iteritems(self): + for key in dict.keys(self): + try: + val = self[key] + except Exception as err: + self.log.warning("Error loading %s: %s" % (key, err)) + continue + yield key, val + + def items(self): + back = [] + for key in dict.keys(self): + try: + val = self[key] + except Exception as err: + self.log.warning("Error loading %s: %s" % (key, err)) + continue + back.append((key, val)) + return back + + def values(self): + back = [] + for key, val in dict.iteritems(self): + if not val: + try: + val = self.loadItem(key) + except Exception: + continue + back.append(val) + return back + + def get(self, key, default=None): + try: + return self.__getitem__(key) + except KeyError: + return default + except Exception as err: + self.site.bad_files[key] = self.site.bad_files.get(key, 1) + dict.__delitem__(self, key) + self.log.warning("Error loading %s: %s" % (key, err)) + return default + + def execute(self, query, params={}): + params["site_id"] = self.db_id + return self.db.execute(query, params) + +if __name__ == "__main__": + import psutil + process = psutil.Process(os.getpid()) + s_mem = process.memory_info()[0] / float(2 ** 20) + root = "data-live/1MaiL5gfBM1cyb4a8e3iiL8L5gXmoAJu27" + contents = ContentDbDict("1MaiL5gfBM1cyb4a8e3iiL8L5gXmoAJu27", root) + print("Init len", len(contents)) + + s = time.time() + for dir_name in os.listdir(root + "/data/users/")[0:8000]: + contents["data/users/%s/content.json" % dir_name] + print("Load: %.3fs" % (time.time() - s)) + + s = time.time() + found = 0 + for key, val in contents.items(): + found += 1 + assert key + assert val + print("Found:", found) + print("Iteritem: %.3fs" % (time.time() - s)) + + s = time.time() + found = 0 + for key in list(contents.keys()): + found += 1 + assert key in contents + print("In: %.3fs" % (time.time() - s)) + + print("Len:", len(list(contents.values())), len(list(contents.keys()))) + + print("Mem: +", process.memory_info()[0] / float(2 ** 20) - s_mem) diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py new file mode 100644 index 00000000..623cc707 --- /dev/null +++ b/src/Content/ContentManager.py @@ -0,0 +1,1067 @@ +import json +import time +import re +import os +import copy +import base64 +import sys + +import gevent + +from Debug import Debug +from Crypt import CryptHash +from Config import config +from util import helper +from util import Diff +from util import SafeRe +from Peer import PeerHashfield +from .ContentDbDict import ContentDbDict +from Plugin import PluginManager + + +class VerifyError(Exception): + pass + + +class SignError(Exception): + pass + + +@PluginManager.acceptPlugins +class ContentManager(object): + + def __init__(self, site): + self.site = site + self.log = self.site.log + self.contents = ContentDbDict(site) + self.hashfield = PeerHashfield() + self.has_optional_files = False + + # Load all content.json files + def loadContents(self): + if len(self.contents) == 0: + self.log.info("ContentDb not initialized, load files from filesystem...") + self.loadContent(add_bad_files=False, delete_removed_files=False) + self.site.settings["size"], self.site.settings["size_optional"] = self.getTotalSize() + + # Load hashfield cache + if "hashfield" in self.site.settings.get("cache", {}): + self.hashfield.frombytes(base64.b64decode(self.site.settings["cache"]["hashfield"])) + del self.site.settings["cache"]["hashfield"] + elif self.contents.get("content.json") and self.site.settings["size_optional"] > 0: + self.site.storage.updateBadFiles() # No hashfield cache created yet + self.has_optional_files = bool(self.hashfield) + + self.contents.db.initSite(self.site) + + def getFileChanges(self, old_files, new_files): + deleted = {key: val for key, val in old_files.items() if key not in new_files} + deleted_hashes = {val.get("sha512"): key for key, val in old_files.items() if key not in new_files} + added = {key: val for key, val in new_files.items() if key not in old_files} + renamed = {} + for relative_path, node in added.items(): + hash = node.get("sha512") + if hash in deleted_hashes: + relative_path_old = deleted_hashes[hash] + renamed[relative_path_old] = relative_path + del(deleted[relative_path_old]) + return list(deleted), renamed + + # Load content.json to self.content + # Return: Changed files ["index.html", "data/messages.json"], Deleted files ["old.jpg"] + def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True, force=False): + content_inner_path = content_inner_path.strip("/") # Remove / from beginning + old_content = self.contents.get(content_inner_path) + content_path = self.site.storage.getPath(content_inner_path) + content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path)) + content_inner_dir = helper.getDirname(content_inner_path) + + if os.path.isfile(content_path): + try: + # Check if file is newer than what we have + if not force and old_content and not self.site.settings.get("own"): + for line in open(content_path): + if '"modified"' not in line: + continue + match = re.search(r"([0-9\.]+),$", line.strip(" \r\n")) + if match and float(match.group(1)) <= old_content.get("modified", 0): + self.log.debug("%s loadContent same json file, skipping" % content_inner_path) + return [], [] + + new_content = self.site.storage.loadJson(content_inner_path) + except Exception as err: + self.log.warning("%s load error: %s" % (content_path, Debug.formatException(err))) + return [], [] + else: + self.log.debug("Content.json not exist: %s" % content_path) + return [], [] # Content.json not exist + + try: + # Get the files where the sha512 changed + changed = [] + deleted = [] + # Check changed + for relative_path, info in new_content.get("files", {}).items(): + if "sha512" in info: + hash_type = "sha512" + else: # Backward compatibility + hash_type = "sha1" + + new_hash = info[hash_type] + if old_content and old_content["files"].get(relative_path): # We have the file in the old content + old_hash = old_content["files"][relative_path].get(hash_type) + else: # The file is not in the old content + old_hash = None + if old_hash != new_hash: + changed.append(content_inner_dir + relative_path) + + # Check changed optional files + for relative_path, info in new_content.get("files_optional", {}).items(): + file_inner_path = content_inner_dir + relative_path + new_hash = info["sha512"] + if old_content and old_content.get("files_optional", {}).get(relative_path): + # We have the file in the old content + old_hash = old_content["files_optional"][relative_path].get("sha512") + if old_hash != new_hash and self.site.isDownloadable(file_inner_path): + changed.append(file_inner_path) # Download new file + elif old_hash != new_hash and self.hashfield.hasHash(old_hash) and not self.site.settings.get("own"): + try: + old_hash_id = self.hashfield.getHashId(old_hash) + self.optionalRemoved(file_inner_path, old_hash_id, old_content["files_optional"][relative_path]["size"]) + self.optionalDelete(file_inner_path) + self.log.debug("Deleted changed optional file: %s" % file_inner_path) + except Exception as err: + self.log.warning("Error deleting file %s: %s" % (file_inner_path, Debug.formatException(err))) + else: # The file is not in the old content + if self.site.isDownloadable(file_inner_path): + changed.append(file_inner_path) # Download new file + + # Check deleted + if old_content: + old_files = dict( + old_content.get("files", {}), + **old_content.get("files_optional", {}) + ) + + new_files = dict( + new_content.get("files", {}), + **new_content.get("files_optional", {}) + ) + + deleted, renamed = self.getFileChanges(old_files, new_files) + + for relative_path_old, relative_path_new in renamed.items(): + self.log.debug("Renaming: %s -> %s" % (relative_path_old, relative_path_new)) + if relative_path_new in new_content.get("files_optional", {}): + self.optionalRenamed(content_inner_dir + relative_path_old, content_inner_dir + relative_path_new) + if self.site.storage.isFile(relative_path_old): + try: + self.site.storage.rename(relative_path_old, relative_path_new) + if relative_path_new in changed: + changed.remove(relative_path_new) + self.log.debug("Renamed: %s -> %s" % (relative_path_old, relative_path_new)) + except Exception as err: + self.log.warning("Error renaming file: %s -> %s %s" % (relative_path_old, relative_path_new, err)) + + if deleted and not self.site.settings.get("own"): + # Deleting files that no longer in content.json + for file_relative_path in deleted: + file_inner_path = content_inner_dir + file_relative_path + try: + # Check if the deleted file is optional + if old_content.get("files_optional") and old_content["files_optional"].get(file_relative_path): + self.optionalDelete(file_inner_path) + old_hash = old_content["files_optional"][file_relative_path].get("sha512") + if self.hashfield.hasHash(old_hash): + old_hash_id = self.hashfield.getHashId(old_hash) + self.optionalRemoved(file_inner_path, old_hash_id, old_content["files_optional"][file_relative_path]["size"]) + else: + self.site.storage.delete(file_inner_path) + + self.log.debug("Deleted file: %s" % file_inner_path) + except Exception as err: + self.log.debug("Error deleting file %s: %s" % (file_inner_path, Debug.formatException(err))) + + # Cleanup empty dirs + tree = {root: [dirs, files] for root, dirs, files in os.walk(self.site.storage.getPath(content_inner_dir))} + for root in sorted(tree, key=len, reverse=True): + dirs, files = tree[root] + if dirs == [] and files == []: + root_inner_path = self.site.storage.getInnerPath(root.replace("\\", "/")) + self.log.debug("Empty directory: %s, cleaning up." % root_inner_path) + try: + self.site.storage.deleteDir(root_inner_path) + # Remove from tree dict to reflect changed state + tree[os.path.dirname(root)][0].remove(os.path.basename(root)) + except Exception as err: + self.log.debug("Error deleting empty directory %s: %s" % (root_inner_path, err)) + + # Check archived + if old_content and "user_contents" in new_content and "archived" in new_content["user_contents"]: + old_archived = old_content.get("user_contents", {}).get("archived", {}) + new_archived = new_content.get("user_contents", {}).get("archived", {}) + self.log.debug("old archived: %s, new archived: %s" % (len(old_archived), len(new_archived))) + archived_changed = { + key: date_archived + for key, date_archived in new_archived.items() + if old_archived.get(key) != new_archived[key] + } + if archived_changed: + self.log.debug("Archived changed: %s" % archived_changed) + for archived_dirname, date_archived in archived_changed.items(): + archived_inner_path = content_inner_dir + archived_dirname + "/content.json" + if self.contents.get(archived_inner_path, {}).get("modified", 0) < date_archived: + self.removeContent(archived_inner_path) + deleted += archived_inner_path + self.site.settings["size"], self.site.settings["size_optional"] = self.getTotalSize() + + # Check archived before + if old_content and "user_contents" in new_content and "archived_before" in new_content["user_contents"]: + old_archived_before = old_content.get("user_contents", {}).get("archived_before", 0) + new_archived_before = new_content.get("user_contents", {}).get("archived_before", 0) + if old_archived_before != new_archived_before: + self.log.debug("Archived before changed: %s -> %s" % (old_archived_before, new_archived_before)) + + # Remove downloaded archived files + num_removed_contents = 0 + for archived_inner_path in self.listModified(before=new_archived_before): + if archived_inner_path.startswith(content_inner_dir) and archived_inner_path != content_inner_path: + self.removeContent(archived_inner_path) + num_removed_contents += 1 + self.site.settings["size"], self.site.settings["size_optional"] = self.getTotalSize() + + # Remove archived files from download queue + num_removed_bad_files = 0 + for bad_file in list(self.site.bad_files.keys()): + if bad_file.endswith("content.json"): + del self.site.bad_files[bad_file] + num_removed_bad_files += 1 + + if num_removed_bad_files > 0: + self.site.worker_manager.removeSolvedFileTasks(mark_as_good=False) + gevent.spawn(self.site.update, since=0) + + self.log.debug("Archived removed contents: %s, removed bad files: %s" % (num_removed_contents, num_removed_bad_files)) + + # Load includes + if load_includes and "includes" in new_content: + for relative_path, info in list(new_content["includes"].items()): + include_inner_path = content_inner_dir + relative_path + if self.site.storage.isFile(include_inner_path): # Content.json exists, load it + include_changed, include_deleted = self.loadContent( + include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files + ) + if include_changed: + changed += include_changed # Add changed files + if include_deleted: + deleted += include_deleted # Add changed files + else: # Content.json not exist, add to changed files + self.log.debug("Missing include: %s" % include_inner_path) + changed += [include_inner_path] + + # Load blind user includes (all subdir) + if load_includes and "user_contents" in new_content: + for relative_dir in os.listdir(content_dir): + include_inner_path = content_inner_dir + relative_dir + "/content.json" + if not self.site.storage.isFile(include_inner_path): + continue # Content.json not exist + include_changed, include_deleted = self.loadContent( + include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files, + load_includes=False + ) + if include_changed: + changed += include_changed # Add changed files + if include_deleted: + deleted += include_deleted # Add changed files + + # Save some memory + new_content["signs"] = None + if "cert_sign" in new_content: + new_content["cert_sign"] = None + + if new_content.get("files_optional"): + self.has_optional_files = True + # Update the content + self.contents[content_inner_path] = new_content + except Exception as err: + self.log.warning("%s parse error: %s" % (content_inner_path, Debug.formatException(err))) + return [], [] # Content.json parse error + + # Add changed files to bad files + if add_bad_files: + for inner_path in changed: + self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1 + for inner_path in deleted: + if inner_path in self.site.bad_files: + del self.site.bad_files[inner_path] + self.site.worker_manager.removeSolvedFileTasks() + + if new_content.get("modified", 0) > self.site.settings.get("modified", 0): + # Dont store modifications in the far future (more than 10 minute) + self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"]) + + return changed, deleted + + def removeContent(self, inner_path): + inner_dir = helper.getDirname(inner_path) + try: + content = self.contents[inner_path] + files = dict( + content.get("files", {}), + **content.get("files_optional", {}) + ) + except Exception as err: + self.log.debug("Error loading %s for removeContent: %s" % (inner_path, Debug.formatException(err))) + files = {} + files["content.json"] = True + # Deleting files that no longer in content.json + for file_relative_path in files: + file_inner_path = inner_dir + file_relative_path + try: + self.site.storage.delete(file_inner_path) + self.log.debug("Deleted file: %s" % file_inner_path) + except Exception as err: + self.log.debug("Error deleting file %s: %s" % (file_inner_path, err)) + try: + self.site.storage.deleteDir(inner_dir) + except Exception as err: + self.log.debug("Error deleting dir %s: %s" % (inner_dir, err)) + + try: + del self.contents[inner_path] + except Exception as err: + self.log.debug("Error key from contents: %s" % inner_path) + + # Get total size of site + # Return: 32819 (size of files in kb) + def getTotalSize(self, ignore=None): + return self.contents.db.getTotalSize(self.site, ignore) + + def listModified(self, after=None, before=None): + return self.contents.db.listModified(self.site, after=after, before=before) + + def listContents(self, inner_path="content.json", user_files=False): + if inner_path not in self.contents: + return [] + back = [inner_path] + content_inner_dir = helper.getDirname(inner_path) + for relative_path in list(self.contents[inner_path].get("includes", {}).keys()): + include_inner_path = content_inner_dir + relative_path + back += self.listContents(include_inner_path) + return back + + # Returns if file with the given modification date is archived or not + def isArchived(self, inner_path, modified): + match = re.match(r"(.*)/(.*?)/", inner_path) + if not match: + return False + user_contents_inner_path = match.group(1) + "/content.json" + relative_directory = match.group(2) + + file_info = self.getFileInfo(user_contents_inner_path) + if file_info: + time_archived_before = file_info.get("archived_before", 0) + time_directory_archived = file_info.get("archived", {}).get(relative_directory, 0) + if modified <= time_archived_before or modified <= time_directory_archived: + return True + else: + return False + else: + return False + + def isDownloaded(self, inner_path, hash_id=None): + if not hash_id: + file_info = self.getFileInfo(inner_path) + if not file_info or "sha512" not in file_info: + return False + hash_id = self.hashfield.getHashId(file_info["sha512"]) + return hash_id in self.hashfield + + # Is modified since signing + def isModified(self, inner_path): + s = time.time() + if inner_path.endswith("content.json"): + try: + is_valid = self.verifyFile(inner_path, self.site.storage.open(inner_path), ignore_same=False) + if is_valid: + is_modified = False + else: + is_modified = True + except VerifyError: + is_modified = True + else: + try: + self.verifyFile(inner_path, self.site.storage.open(inner_path), ignore_same=False) + is_modified = False + except VerifyError: + is_modified = True + return is_modified + + # Find the file info line from self.contents + # Return: { "sha512": "c29d73d...21f518", "size": 41 , "content_inner_path": "content.json"} + def getFileInfo(self, inner_path, new_file=False): + dirs = inner_path.split("/") # Parent dirs of content.json + inner_path_parts = [dirs.pop()] # Filename relative to content.json + while True: + content_inner_path = "%s/content.json" % "/".join(dirs) + content_inner_path = content_inner_path.strip("/") + content = self.contents.get(content_inner_path) + + # Check in files + if content and "files" in content: + back = content["files"].get("/".join(inner_path_parts)) + if back: + back["content_inner_path"] = content_inner_path + back["optional"] = False + back["relative_path"] = "/".join(inner_path_parts) + return back + + # Check in optional files + if content and "files_optional" in content: # Check if file in this content.json + back = content["files_optional"].get("/".join(inner_path_parts)) + if back: + back["content_inner_path"] = content_inner_path + back["optional"] = True + back["relative_path"] = "/".join(inner_path_parts) + return back + + # Return the rules if user dir + if content and "user_contents" in content: + back = content["user_contents"] + content_inner_path_dir = helper.getDirname(content_inner_path) + relative_content_path = inner_path[len(content_inner_path_dir):] + user_auth_address_match = re.match(r"([A-Za-z0-9]+)/.*", relative_content_path) + if user_auth_address_match: + user_auth_address = user_auth_address_match.group(1) + back["content_inner_path"] = "%s%s/content.json" % (content_inner_path_dir, user_auth_address) + else: + back["content_inner_path"] = content_inner_path_dir + "content.json" + back["optional"] = None + back["relative_path"] = "/".join(inner_path_parts) + return back + + if new_file and content: + back = {} + back["content_inner_path"] = content_inner_path + back["relative_path"] = "/".join(inner_path_parts) + back["optional"] = None + return back + + # No inner path in this dir, lets try the parent dir + if dirs: + inner_path_parts.insert(0, dirs.pop()) + else: # No more parent dirs + break + + # Not found + return False + + # Get rules for the file + # Return: The rules for the file or False if not allowed + def getRules(self, inner_path, content=None): + if not inner_path.endswith("content.json"): # Find the files content.json first + file_info = self.getFileInfo(inner_path) + if not file_info: + return False # File not found + inner_path = file_info["content_inner_path"] + + if inner_path == "content.json": # Root content.json + rules = {} + rules["signers"] = self.getValidSigners(inner_path, content) + return rules + + dirs = inner_path.split("/") # Parent dirs of content.json + inner_path_parts = [dirs.pop()] # Filename relative to content.json + inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir + while True: + content_inner_path = "%s/content.json" % "/".join(dirs) + parent_content = self.contents.get(content_inner_path.strip("/")) + if parent_content and "includes" in parent_content: + return parent_content["includes"].get("/".join(inner_path_parts)) + elif parent_content and "user_contents" in parent_content: + return self.getUserContentRules(parent_content, inner_path, content) + else: # No inner path in this dir, lets try the parent dir + if dirs: + inner_path_parts.insert(0, dirs.pop()) + else: # No more parent dirs + break + + return False + + # Get rules for a user file + # Return: The rules of the file or False if not allowed + def getUserContentRules(self, parent_content, inner_path, content): + user_contents = parent_content["user_contents"] + + # Delivered for directory + if "inner_path" in parent_content: + parent_content_dir = helper.getDirname(parent_content["inner_path"]) + user_address = re.match(r"([A-Za-z0-9]*?)/", inner_path[len(parent_content_dir):]).group(1) + else: + user_address = re.match(r".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) + + try: + if not content: + content = self.site.storage.loadJson(inner_path) # Read the file if no content specified + user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit + cert_user_id = content["cert_user_id"] + except Exception: # Content.json not exist + user_urn = "n-a/n-a" + cert_user_id = "n-a" + + if user_address in user_contents["permissions"]: + rules = copy.copy(user_contents["permissions"].get(user_address, {})) # Default rules based on address + else: + rules = copy.copy(user_contents["permissions"].get(cert_user_id, {})) # Default rules based on username + + if rules is False: + banned = True + rules = {} + else: + banned = False + if "signers" in rules: + rules["signers"] = rules["signers"][:] # Make copy of the signers + for permission_pattern, permission_rules in list(user_contents["permission_rules"].items()): # Regexp rules + if not SafeRe.match(permission_pattern, user_urn): + continue # Rule is not valid for user + # Update rules if its better than current recorded ones + for key, val in permission_rules.items(): + if key not in rules: + if type(val) is list: + rules[key] = val[:] # Make copy + else: + rules[key] = val + elif type(val) is int: # Int, update if larger + if val > rules[key]: + rules[key] = val + elif hasattr(val, "startswith"): # String, update if longer + if len(val) > len(rules[key]): + rules[key] = val + elif type(val) is list: # List, append + rules[key] += val + + # Accepted cert signers + rules["cert_signers"] = user_contents.get("cert_signers", {}) + rules["cert_signers_pattern"] = user_contents.get("cert_signers_pattern") + + if "signers" not in rules: + rules["signers"] = [] + + if not banned: + rules["signers"].append(user_address) # Add user as valid signer + rules["user_address"] = user_address + rules["includes_allowed"] = False + + return rules + + # Get diffs for changed files + def getDiffs(self, inner_path, limit=30 * 1024, update_files=True): + if inner_path not in self.contents: + return {} + diffs = {} + content_inner_path_dir = helper.getDirname(inner_path) + for file_relative_path in self.contents[inner_path].get("files", {}): + file_inner_path = content_inner_path_dir + file_relative_path + if self.site.storage.isFile(file_inner_path + "-new"): # New version present + diffs[file_relative_path] = Diff.diff( + list(self.site.storage.open(file_inner_path)), + list(self.site.storage.open(file_inner_path + "-new")), + limit=limit + ) + if update_files: + self.site.storage.delete(file_inner_path) + self.site.storage.rename(file_inner_path + "-new", file_inner_path) + if self.site.storage.isFile(file_inner_path + "-old"): # Old version present + diffs[file_relative_path] = Diff.diff( + list(self.site.storage.open(file_inner_path + "-old")), + list(self.site.storage.open(file_inner_path)), + limit=limit + ) + if update_files: + self.site.storage.delete(file_inner_path + "-old") + return diffs + + def hashFile(self, dir_inner_path, file_relative_path, optional=False): + back = {} + file_inner_path = dir_inner_path + "/" + file_relative_path + + file_path = self.site.storage.getPath(file_inner_path) + file_size = os.path.getsize(file_path) + sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file + if optional and not self.hashfield.hasHash(sha512sum): + self.optionalDownloaded(file_inner_path, self.hashfield.getHashId(sha512sum), file_size, own=True) + + back[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)} + return back + + def isValidRelativePath(self, relative_path): + if ".." in relative_path.replace("\\", "/").split("/"): + return False + elif len(relative_path) > 255: + return False + elif relative_path[0] in ("/", "\\"): # Starts with + return False + elif relative_path[-1] in (".", " "): # Ends with + return False + elif re.match(r".*(^|/)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9]|CONOUT\$|CONIN\$)(\.|/|$)", relative_path, re.IGNORECASE): # Protected on Windows + return False + else: + return re.match(r"^[^\x00-\x1F\"*:<>?\\|]+$", relative_path) + + def sanitizePath(self, inner_path): + return re.sub("[\x00-\x1F\"*:<>?\\|]", "", inner_path) + + # Hash files in directory + def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None): + files_node = {} + files_optional_node = {} + db_inner_path = self.site.storage.getDbFile() + if dir_inner_path and not self.isValidRelativePath(dir_inner_path): + ignored = True + self.log.error("- [ERROR] Only ascii encoded directories allowed: %s" % dir_inner_path) + + for file_relative_path in self.site.storage.walk(dir_inner_path, ignore_pattern): + file_name = helper.getFilename(file_relative_path) + + ignored = optional = False + if file_name == "content.json": + ignored = True + elif file_name.startswith(".") or file_name.endswith("-old") or file_name.endswith("-new"): + ignored = True + elif not self.isValidRelativePath(file_relative_path): + ignored = True + self.log.error("- [ERROR] Invalid filename: %s" % file_relative_path) + elif dir_inner_path == "" and db_inner_path and file_relative_path.startswith(db_inner_path): + ignored = True + elif optional_pattern and SafeRe.match(optional_pattern, file_relative_path): + optional = True + + if ignored: # Ignore content.json, defined regexp and files starting with . + self.log.info("- [SKIPPED] %s" % file_relative_path) + else: + if optional: + self.log.info("- [OPTIONAL] %s" % file_relative_path) + files_optional_node.update( + self.hashFile(dir_inner_path, file_relative_path, optional=True) + ) + else: + self.log.info("- %s" % file_relative_path) + files_node.update( + self.hashFile(dir_inner_path, file_relative_path) + ) + return files_node, files_optional_node + + # Create and sign a content.json + # Return: The new content if filewrite = False + def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None, remove_missing_optional=False): + if not inner_path.endswith("content.json"): + raise SignError("Invalid file name, you can only sign content.json files") + + if inner_path in self.contents: + content = self.contents.get(inner_path) + if content and content.get("cert_sign", False) is None and self.site.storage.isFile(inner_path): + # Recover cert_sign from file + content["cert_sign"] = self.site.storage.loadJson(inner_path).get("cert_sign") + else: + content = None + if not content: # Content not exist yet, load default one + self.log.info("File %s not exist yet, loading default values..." % inner_path) + + if self.site.storage.isFile(inner_path): + content = self.site.storage.loadJson(inner_path) + if "files" not in content: + content["files"] = {} + if "signs" not in content: + content["signs"] = {} + else: + content = {"files": {}, "signs": {}} # Default content.json + + if inner_path == "content.json": # It's the root content.json, add some more fields + content["title"] = "%s - ZeroNet_" % self.site.address + content["description"] = "" + content["signs_required"] = 1 + content["ignore"] = "" + + if extend: + # Add extend keys if not exists + for key, val in list(extend.items()): + if not content.get(key): + content[key] = val + self.log.info("Extending content.json with: %s" % key) + + directory = helper.getDirname(self.site.storage.getPath(inner_path)) + inner_directory = helper.getDirname(inner_path) + self.log.info("Opening site data directory: %s..." % directory) + + changed_files = [inner_path] + files_node, files_optional_node = self.hashFiles( + helper.getDirname(inner_path), content.get("ignore"), content.get("optional") + ) + + if not remove_missing_optional: + for file_inner_path, file_details in content.get("files_optional", {}).items(): + if file_inner_path not in files_optional_node: + files_optional_node[file_inner_path] = file_details + + # Find changed files + files_merged = files_node.copy() + files_merged.update(files_optional_node) + for file_relative_path, file_details in files_merged.items(): + old_hash = content.get("files", {}).get(file_relative_path, {}).get("sha512") + new_hash = files_merged[file_relative_path]["sha512"] + if old_hash != new_hash: + changed_files.append(inner_directory + file_relative_path) + + self.log.debug("Changed files: %s" % changed_files) + if update_changed_files: + for file_path in changed_files: + self.site.storage.onUpdated(file_path) + + # Generate new content.json + self.log.info("Adding timestamp and sha512sums to new content.json...") + + new_content = content.copy() # Create a copy of current content.json + new_content["files"] = files_node # Add files sha512 hash + if files_optional_node: + new_content["files_optional"] = files_optional_node + elif "files_optional" in new_content: + del new_content["files_optional"] + + if inner_path == "content.json": + new_content["zeronet_version"] = config.version + new_content["signs_required"] = content.get("signs_required", 1) + + new_content["address"] = self.site.address + new_content["inner_path"] = inner_path + + # Verify private key + from Crypt import CryptBitcoin + self.log.info("Verifying private key...") + privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey) + valid_signers = self.getValidSigners(inner_path, new_content) + if privatekey_address not in valid_signers: + raise SignError( + "Private key invalid! Valid signers: %s, Private key address: %s" % + (valid_signers, privatekey_address) + ) + self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers)) + + signs_required = 1 + if inner_path == "content.json" and privatekey_address == self.site.address: + # If signing using the root key, then sign the valid signers + signs_required = new_content["signs_required"] + signers_data = "%s:%s" % (signs_required, ",".join(valid_signers)) + new_content["signers_sign"] = CryptBitcoin.sign(str(signers_data), privatekey) + if not new_content["signers_sign"]: + self.log.info("Old style address, signers_sign is none") + + self.log.info("Signing %s..." % inner_path) + + if "signs" in new_content: + # del(new_content["signs"]) # Delete old signs + old_signs_content = new_content["signs"] + del(new_content["signs"]) + else: + old_signs_content = None + if "sign" in new_content: + del(new_content["sign"]) # Delete old sign (backward compatibility) + + if signs_required > 1: + has_valid_sign = False + sign_content = json.dumps(new_content, sort_keys=True) + for signer in valid_signers: + res = CryptBitcoin.verify(sign_content,signer,old_signs_content[signer]); + print(res) + if res: + has_valid_sign = has_valid_sign or res + if has_valid_sign: + new_content["modified"] = content["modified"] + sign_content = json.dumps(new_content, sort_keys=True) + else: + new_content["modified"] = int(time.time()) # Add timestamp + sign_content = json.dumps(new_content, sort_keys=True) + sign = CryptBitcoin.sign(sign_content, privatekey) + # new_content["signs"] = content.get("signs", {}) # TODO: Multisig + if sign: # If signing is successful (not an old address) + new_content["signs"] = old_signs_content or {} + new_content["signs"][privatekey_address] = sign + + self.verifyContent(inner_path, new_content) + + if filewrite: + self.log.info("Saving to %s..." % inner_path) + self.site.storage.writeJson(inner_path, new_content) + self.contents[inner_path] = new_content + + self.log.info("File %s signed!" % inner_path) + + if filewrite: # Written to file + return True + else: # Return the new content + return new_content + + # The valid signers of content.json file + # Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"] + def getValidSigners(self, inner_path, content=None): + valid_signers = [] + if inner_path == "content.json": # Root content.json + if "content.json" in self.contents and "signers" in self.contents["content.json"]: + valid_signers += self.contents["content.json"]["signers"][:] + else: + rules = self.getRules(inner_path, content) + if rules and "signers" in rules: + valid_signers += rules["signers"] + + if self.site.address not in valid_signers: + valid_signers.append(self.site.address) # Site address always valid + return valid_signers + + # Return: The required number of valid signs for the content.json + def getSignsRequired(self, inner_path, content=None): + if not content: + return 1 + return content.get("signs_required", 1) + + def verifyCertSign(self, user_address, user_auth_type, user_name, issuer_address, sign): + from Crypt import CryptBitcoin + cert_subject = "%s#%s/%s" % (user_address, user_auth_type, user_name) + return CryptBitcoin.verify(cert_subject, issuer_address, sign) + + def verifyCert(self, inner_path, content): + rules = self.getRules(inner_path, content) + + if not rules: + raise VerifyError("No rules for this file") + + if not rules.get("cert_signers") and not rules.get("cert_signers_pattern"): + return True # Does not need cert + + if "cert_user_id" not in content: + raise VerifyError("Missing cert_user_id") + + if content["cert_user_id"].count("@") != 1: + raise VerifyError("Invalid domain in cert_user_id") + + name, domain = content["cert_user_id"].rsplit("@", 1) + cert_address = rules["cert_signers"].get(domain) + if not cert_address: # Unknown Cert signer + if rules.get("cert_signers_pattern") and SafeRe.match(rules["cert_signers_pattern"], domain): + cert_address = domain + else: + raise VerifyError("Invalid cert signer: %s" % domain) + + return self.verifyCertSign(rules["user_address"], content["cert_auth_type"], name, cert_address, content["cert_sign"]) + + # Checks if the content.json content is valid + # Return: True or False + def verifyContent(self, inner_path, content): + content_size = len(json.dumps(content, indent=1)) + sum([file["size"] for file in list(content["files"].values()) if file["size"] >= 0]) # Size of new content + # Calculate old content size + old_content = self.contents.get(inner_path) + if old_content: + old_content_size = len(json.dumps(old_content, indent=1)) + sum([file["size"] for file in list(old_content.get("files", {}).values())]) + old_content_size_optional = sum([file["size"] for file in list(old_content.get("files_optional", {}).values())]) + else: + old_content_size = 0 + old_content_size_optional = 0 + + # Reset site site on first content.json + if not old_content and inner_path == "content.json": + self.site.settings["size"] = 0 + + content_size_optional = sum([file["size"] for file in list(content.get("files_optional", {}).values()) if file["size"] >= 0]) + site_size = self.site.settings["size"] - old_content_size + content_size # Site size without old content plus the new + site_size_optional = self.site.settings["size_optional"] - old_content_size_optional + content_size_optional # Site size without old content plus the new + + site_size_limit = self.site.getSizeLimit() * 1024 * 1024 + + # Check site address + if content.get("address") and content["address"] != self.site.address: + raise VerifyError("Wrong site address: %s != %s" % (content["address"], self.site.address)) + + # Check file inner path + if content.get("inner_path") and content["inner_path"] != inner_path: + raise VerifyError("Wrong inner_path: %s" % content["inner_path"]) + + # If our content.json file bigger than the size limit throw error + if inner_path == "content.json": + content_size_file = len(json.dumps(content, indent=1)) + if content_size_file > site_size_limit: + # Save site size to display warning + self.site.settings["size"] = site_size + task = self.site.worker_manager.tasks.findTask(inner_path) + if task: # Dont try to download from other peers + self.site.worker_manager.failTask(task) + raise VerifyError("Content too large %s B > %s B, aborting task..." % (site_size, site_size_limit)) + + # Verify valid filenames + for file_relative_path in list(content.get("files", {}).keys()) + list(content.get("files_optional", {}).keys()): + if not self.isValidRelativePath(file_relative_path): + raise VerifyError("Invalid relative path: %s" % file_relative_path) + + if inner_path == "content.json": + self.site.settings["size"] = site_size + self.site.settings["size_optional"] = site_size_optional + return True # Root content.json is passed + else: + if self.verifyContentInclude(inner_path, content, content_size, content_size_optional): + self.site.settings["size"] = site_size + self.site.settings["size_optional"] = site_size_optional + return True + else: + raise VerifyError("Content verify error") + + def verifyContentInclude(self, inner_path, content, content_size, content_size_optional): + # Load include details + rules = self.getRules(inner_path, content) + if not rules: + raise VerifyError("No rules") + + # Check include size limit + if rules.get("max_size") is not None: # Include size limit + if content_size > rules["max_size"]: + raise VerifyError("Include too large %sB > %sB" % (content_size, rules["max_size"])) + + if rules.get("max_size_optional") is not None: # Include optional files limit + if content_size_optional > rules["max_size_optional"]: + raise VerifyError("Include optional files too large %sB > %sB" % ( + content_size_optional, rules["max_size_optional"]) + ) + + # Filename limit + if rules.get("files_allowed"): + for file_inner_path in list(content["files"].keys()): + if not SafeRe.match(r"^%s$" % rules["files_allowed"], file_inner_path): + raise VerifyError("File not allowed: %s" % file_inner_path) + + if rules.get("files_allowed_optional"): + for file_inner_path in list(content.get("files_optional", {}).keys()): + if not SafeRe.match(r"^%s$" % rules["files_allowed_optional"], file_inner_path): + raise VerifyError("Optional file not allowed: %s" % file_inner_path) + + # Check if content includes allowed + if rules.get("includes_allowed") is False and content.get("includes"): + raise VerifyError("Includes not allowed") + + return True # All good + + # Verify file validity + # Return: None = Same as before, False = Invalid, True = Valid + def verifyFile(self, inner_path, file, ignore_same=True): + if inner_path.endswith("content.json"): # content.json: Check using sign + from Crypt import CryptBitcoin + try: + if type(file) is dict: + new_content = file + else: + try: + if sys.version_info.major == 3 and sys.version_info.minor < 6: + new_content = json.loads(file.read().decode("utf8")) + else: + new_content = json.load(file) + except Exception as err: + raise VerifyError("Invalid json file: %s" % err) + if inner_path in self.contents: + old_content = self.contents.get(inner_path, {"modified": 0}) + # Checks if its newer the ours + if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json + return None + elif old_content["modified"] > new_content["modified"]: # We have newer + raise VerifyError( + "We have newer (Our: %s, Sent: %s)" % + (old_content["modified"], new_content["modified"]) + ) + if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+) + raise VerifyError("Modify timestamp is in the far future!") + if self.isArchived(inner_path, new_content["modified"]): + if inner_path in self.site.bad_files: + del self.site.bad_files[inner_path] + raise VerifyError("This file is archived!") + # Check sign + sign = new_content.get("sign") + signs = new_content.get("signs", {}) + if "sign" in new_content: + del(new_content["sign"]) # The file signed without the sign + if "signs" in new_content: + del(new_content["signs"]) # The file signed without the signs + + sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace + + # Fix float representation error on Android + modified = new_content["modified"] + if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"): + modified_fixed = "{:.6f}".format(modified).strip("0.") + sign_content = sign_content.replace( + '"modified": %s' % repr(modified), + '"modified": %s' % modified_fixed + ) + + if signs: # New style signing + valid_signers = self.getValidSigners(inner_path, new_content) + signs_required = self.getSignsRequired(inner_path, new_content) + + if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json + signers_data = "%s:%s" % (signs_required, ",".join(valid_signers)) + if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]): + raise VerifyError("Invalid signers_sign!") + + if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid + raise VerifyError("Invalid cert!") + + valid_signs = [] + for address in valid_signers: + if address in signs: + result = CryptBitcoin.verify(sign_content, address, signs[address]) + if result: + valid_signs.append(address) + if len(valid_signs) >= signs_required: + break # Break if we has enough signs + if len(valid_signs) < signs_required: + raise VerifyError("Valid signs: %s/%s, Valid Signers : %s" % (len(valid_signs), signs_required, valid_signs)) + else: + return self.verifyContent(inner_path, new_content) + else: # Old style signing + raise VerifyError("Invalid old-style sign") + + except Exception as err: + self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err))) + raise err + + else: # Check using sha512 hash + file_info = self.getFileInfo(inner_path) + if file_info: + if CryptHash.sha512sum(file) != file_info.get("sha512", ""): + raise VerifyError("Invalid hash") + + if file_info.get("size", 0) != file.tell(): + raise VerifyError( + "File size does not match %s <> %s" % + (inner_path, file.tell(), file_info.get("size", 0)) + ) + + return True + + else: # File not in content.json + raise VerifyError("File not in content.json") + + def optionalDelete(self, inner_path): + self.site.storage.delete(inner_path) + + def optionalDownloaded(self, inner_path, hash_id, size=None, own=False): + if size is None: + size = self.site.storage.getSize(inner_path) + + done = self.hashfield.appendHashId(hash_id) + self.site.settings["optional_downloaded"] += size + return done + + def optionalRemoved(self, inner_path, hash_id, size=None): + if size is None: + size = self.site.storage.getSize(inner_path) + done = self.hashfield.removeHashId(hash_id) + + self.site.settings["optional_downloaded"] -= size + return done + + def optionalRenamed(self, inner_path_old, inner_path_new): + return True diff --git a/src/Content/__init__.py b/src/Content/__init__.py new file mode 100644 index 00000000..fbbd39f4 --- /dev/null +++ b/src/Content/__init__.py @@ -0,0 +1 @@ +from .ContentManager import ContentManager \ No newline at end of file diff --git a/src/Crypt/Crypt.py b/src/Crypt/Crypt.py new file mode 100644 index 00000000..7d7d3659 --- /dev/null +++ b/src/Crypt/Crypt.py @@ -0,0 +1,4 @@ +from Config import config +from util import ThreadPool + +thread_pool_crypt = ThreadPool.ThreadPool(config.threads_crypt) \ No newline at end of file diff --git a/src/Crypt/CryptBitcoin.py b/src/Crypt/CryptBitcoin.py new file mode 100644 index 00000000..68b2caa2 --- /dev/null +++ b/src/Crypt/CryptBitcoin.py @@ -0,0 +1,101 @@ +import logging +import base64 +import binascii +import time +import hashlib + +from util.Electrum import dbl_format +from Config import config + +import util.OpensslFindPatch + +lib_verify_best = "sslcrypto" + +from lib import sslcrypto +sslcurve_native = sslcrypto.ecc.get_curve("secp256k1") +sslcurve_fallback = sslcrypto.fallback.ecc.get_curve("secp256k1") +sslcurve = sslcurve_native + +def loadLib(lib_name, silent=False): + global sslcurve, libsecp256k1message, lib_verify_best + if lib_name == "libsecp256k1": + s = time.time() + from lib import libsecp256k1message + import coincurve + lib_verify_best = "libsecp256k1" + if not silent: + logging.info( + "Libsecpk256k1 loaded: %s in %.3fs" % + (type(coincurve._libsecp256k1.lib).__name__, time.time() - s) + ) + elif lib_name == "sslcrypto": + sslcurve = sslcurve_native + if sslcurve_native == sslcurve_fallback: + logging.warning("SSLCurve fallback loaded instead of native") + elif lib_name == "sslcrypto_fallback": + sslcurve = sslcurve_fallback + +try: + if not config.use_libsecp256k1: + raise Exception("Disabled by config") + loadLib("libsecp256k1") + lib_verify_best = "libsecp256k1" +except Exception as err: + logging.info("Libsecp256k1 load failed: %s" % err) + + +def newPrivatekey(): # Return new private key + return sslcurve.private_to_wif(sslcurve.new_private_key()).decode() + + +def newSeed(): + return binascii.hexlify(sslcurve.new_private_key()).decode() + + +def hdPrivatekey(seed, child): + # Too large child id could cause problems + privatekey_bin = sslcurve.derive_child(seed.encode(), child % 100000000) + return sslcurve.private_to_wif(privatekey_bin).decode() + + +def privatekeyToAddress(privatekey): # Return address from private key + try: + if len(privatekey) == 64: + privatekey_bin = bytes.fromhex(privatekey) + else: + privatekey_bin = sslcurve.wif_to_private(privatekey.encode()) + return sslcurve.private_to_address(privatekey_bin).decode() + except Exception: # Invalid privatekey + return False + + +def sign(data, privatekey): # Return sign to data using private key + if privatekey.startswith("23") and len(privatekey) > 52: + return None # Old style private key not supported + return base64.b64encode(sslcurve.sign( + data.encode(), + sslcurve.wif_to_private(privatekey.encode()), + recoverable=True, + hash=dbl_format + )).decode() + + +def verify(data, valid_address, sign, lib_verify=None): # Verify data using address and sign + if not lib_verify: + lib_verify = lib_verify_best + + if not sign: + return False + + if lib_verify == "libsecp256k1": + sign_address = libsecp256k1message.recover_address(data.encode("utf8"), sign).decode("utf8") + elif lib_verify in ("sslcrypto", "sslcrypto_fallback"): + publickey = sslcurve.recover(base64.b64decode(sign), data.encode(), hash=dbl_format) + sign_address = sslcurve.public_to_address(publickey).decode() + else: + raise Exception("No library enabled for signature verification") + + if type(valid_address) is list: # Any address in the list + return sign_address in valid_address + else: # One possible address + return sign_address == valid_address diff --git a/CryptConnection.py b/src/Crypt/CryptConnection.py similarity index 86% rename from CryptConnection.py rename to src/Crypt/CryptConnection.py index f3c6b7b6..c0903e84 100644 --- a/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -11,13 +11,12 @@ from util import helper class CryptConnectionManager: def __init__(self): - this_file = os.path.abspath(__file__).replace("\\", "/").rstrip("cd") - if sys.platform.startswith("win"): + if config.openssl_bin_file: + self.openssl_bin = config.openssl_bin_file + elif sys.platform.startswith("win"): self.openssl_bin = "tools\\openssl\\openssl.exe" elif config.dist_type.startswith("bundle_linux"): self.openssl_bin = "../runtime/bin/openssl" - elif "in.canews.zeronet" in this_file: - self.openssl_bin = "../usr/bin/openssl" else: self.openssl_bin = "openssl" @@ -91,17 +90,13 @@ class CryptConnectionManager: def wrapSocket(self, sock, crypt, server=False, cert_pin=None): if crypt == "tls-rsa": if server: - sock_wrapped = self.context_server.wrap_socket( - sock, server_side=True) + sock_wrapped = self.context_server.wrap_socket(sock, server_side=True) else: - sock_wrapped = self.context_client.wrap_socket( - sock, server_hostname=random.choice(self.fakedomains)) + sock_wrapped = self.context_client.wrap_socket(sock, server_hostname=random.choice(self.fakedomains)) if cert_pin: - cert_hash = hashlib.sha256( - sock_wrapped.getpeercert(True)).hexdigest() + cert_hash = hashlib.sha256(sock_wrapped.getpeercert(True)).hexdigest() if cert_hash != cert_pin: - raise Exception( - "Socket certificate does not match (%s != %s)" % (cert_hash, cert_pin)) + raise Exception("Socket certificate does not match (%s != %s)" % (cert_hash, cert_pin)) return sock_wrapped else: return sock @@ -132,6 +127,10 @@ class CryptConnectionManager: "/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Domain Validation Secure Server CA" ] self.openssl_env['CN'] = random.choice(self.fakedomains) + environ = os.environ + environ['OPENSSL_CONF'] = self.openssl_env['OPENSSL_CONF'] + environ['RANDFILE'] = self.openssl_env['RANDFILE'] + environ['CN'] = self.openssl_env['CN'] if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem): self.createSslContexts() @@ -141,8 +140,7 @@ class CryptConnectionManager: # Replace variables in config template conf_template = open(self.openssl_conf_template).read() - conf_template = conf_template.replace( - "$ENV::CN", self.openssl_env['CN']) + conf_template = conf_template.replace("$ENV::CN", self.openssl_env['CN']) open(self.openssl_conf, "w").write(conf_template) # Generate CAcert and CAkey @@ -158,14 +156,13 @@ class CryptConnectionManager: self.log.debug("Running: %s" % cmd) proc = subprocess.Popen( cmd, shell=True, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, env=self.openssl_env + stdout=subprocess.PIPE, env=environ ) back = proc.stdout.read().strip().decode(errors="replace").replace("\r", "") proc.wait() if not (os.path.isfile(self.cacert_pem) and os.path.isfile(self.cakey_pem)): - self.log.error( - "RSA ECC SSL CAcert generation failed, CAcert or CAkey files not exist. (%s)" % back) + self.log.error("RSA ECC SSL CAcert generation failed, CAcert or CAkey files not exist. (%s)" % back) return False else: self.log.debug("Result: %s" % back) @@ -182,7 +179,7 @@ class CryptConnectionManager: self.log.debug("Generating certificate key and signing request...") proc = subprocess.Popen( cmd, shell=True, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, env=self.openssl_env + stdout=subprocess.PIPE, env=environ ) back = proc.stdout.read().strip().decode(errors="replace").replace("\r", "") proc.wait() @@ -201,7 +198,7 @@ class CryptConnectionManager: self.log.debug("Generating RSA cert...") proc = subprocess.Popen( cmd, shell=True, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, env=self.openssl_env + stdout=subprocess.PIPE, env=environ ) back = proc.stdout.read().strip().decode(errors="replace").replace("\r", "") proc.wait() @@ -218,8 +215,7 @@ class CryptConnectionManager: return True else: - self.log.error( - "RSA ECC SSL cert generation failed, cert or key files not exist.") + self.log.error("RSA ECC SSL cert generation failed, cert or key files not exist.") manager = CryptConnectionManager() diff --git a/src/Crypt/CryptHash.py b/src/Crypt/CryptHash.py new file mode 100644 index 00000000..f5901fb8 --- /dev/null +++ b/src/Crypt/CryptHash.py @@ -0,0 +1,56 @@ +import hashlib +import os +import base64 + + +def sha512sum(file, blocksize=65536, format="hexdigest"): + if type(file) is str: # Filename specified + file = open(file, "rb") + hash = hashlib.sha512() + for block in iter(lambda: file.read(blocksize), b""): + hash.update(block) + + # Truncate to 256bits is good enough + if format == "hexdigest": + return hash.hexdigest()[0:64] + else: + return hash.digest()[0:32] + + +def sha256sum(file, blocksize=65536): + if type(file) is str: # Filename specified + file = open(file, "rb") + hash = hashlib.sha256() + for block in iter(lambda: file.read(blocksize), b""): + hash.update(block) + return hash.hexdigest() + + +def random(length=64, encoding="hex"): + if encoding == "base64": # Characters: A-Za-z0-9 + hash = hashlib.sha512(os.urandom(256)).digest() + return base64.b64encode(hash).decode("ascii").replace("+", "").replace("/", "").replace("=", "")[0:length] + else: # Characters: a-f0-9 (faster) + return hashlib.sha512(os.urandom(256)).hexdigest()[0:length] + + +# Sha512 truncated to 256bits +class Sha512t: + def __init__(self, data): + if data: + self.sha512 = hashlib.sha512(data) + else: + self.sha512 = hashlib.sha512() + + def hexdigest(self): + return self.sha512.hexdigest()[0:64] + + def digest(self): + return self.sha512.digest()[0:32] + + def update(self, data): + return self.sha512.update(data) + + +def sha512t(data=None): + return Sha512t(data) diff --git a/src/Crypt/CryptTor.py b/src/Crypt/CryptTor.py new file mode 100644 index 00000000..78ba6fc2 --- /dev/null +++ b/src/Crypt/CryptTor.py @@ -0,0 +1,85 @@ +import base64 +import hashlib + +def sign(data, privatekey): + import rsa + from rsa import pkcs1 + from lib import Ed25519 + + ## Onion Service V3 + if len(privatekey) == 88: + prv_key = base64.b64decode(privatekey) + pub_key = Ed25519.publickey_unsafe(prv_key) + sign = Ed25519.signature_unsafe(data, prv_key, pub_key) + + return sign + + ## Onion Service V2 + if "BEGIN RSA PRIVATE KEY" not in privatekey: + privatekey = "-----BEGIN RSA PRIVATE KEY-----\n%s\n-----END RSA PRIVATE KEY-----" % privatekey + + priv = rsa.PrivateKey.load_pkcs1(privatekey) + sign = rsa.pkcs1.sign(data, priv, 'SHA-256') + return sign + +def verify(data, publickey, sign): + import rsa + from rsa import pkcs1 + from lib import Ed25519 + + ## Onion Service V3 + if len(publickey) == 32: + + try: + valid = Ed25519.checkvalid(sign, data, publickey) + valid = 'SHA-256' + + except Exception as err: + print(err) + valid = False + + return valid + + ## Onion Service V2 + pub = rsa.PublicKey.load_pkcs1(publickey, format="DER") + + try: + valid = rsa.pkcs1.verify(data, sign, pub) + + except pkcs1.VerificationError: + valid = False + + return valid + +def privatekeyToPublickey(privatekey): + import rsa + from rsa import pkcs1 + from lib import Ed25519 + + ## Onion Service V3 + if len(privatekey) == 88: + prv_key = base64.b64decode(privatekey) + pub_key = Ed25519.publickey_unsafe(prv_key) + + return pub_key + + ## Onion Service V2 + if "BEGIN RSA PRIVATE KEY" not in privatekey: + privatekey = "-----BEGIN RSA PRIVATE KEY-----\n%s\n-----END RSA PRIVATE KEY-----" % privatekey + + priv = rsa.PrivateKey.load_pkcs1(privatekey) + pub = rsa.PublicKey(priv.n, priv.e) + + return pub.save_pkcs1("DER") + +def publickeyToOnion(publickey): + from lib import Ed25519 + + ## Onion Service V3 + if len(publickey) == 32: + addr = Ed25519.publickey_to_onionaddress(publickey)[:-6] + + return addr + + ## Onion Service V2 + return base64.b32encode(hashlib.sha1(publickey).digest()[:10]).lower().decode("ascii") diff --git a/src/Crypt/__init__.py b/src/Crypt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Db/Db.py b/src/Db/Db.py new file mode 100644 index 00000000..d1d9ce15 --- /dev/null +++ b/src/Db/Db.py @@ -0,0 +1,519 @@ +import sqlite3 +import json +import time +import logging +import re +import os +import atexit +import threading +import sys +import weakref +import errno + +import gevent + +from Debug import Debug +from .DbCursor import DbCursor +from util import SafeRe +from util import helper +from util import ThreadPool +from Config import config + +thread_pool_db = ThreadPool.ThreadPool(config.threads_db) + +next_db_id = 0 +opened_dbs = [] + + +# Close idle databases to save some memory +def dbCleanup(): + while 1: + time.sleep(60 * 5) + for db in opened_dbs[:]: + idle = time.time() - db.last_query_time + if idle > 60 * 5 and db.close_idle: + db.close("Cleanup") + + +def dbCommitCheck(): + while 1: + time.sleep(5) + for db in opened_dbs[:]: + if not db.need_commit: + continue + + success = db.commit("Interval") + if success: + db.need_commit = False + time.sleep(0.1) + + +def dbCloseAll(): + for db in opened_dbs[:]: + db.close("Close all") + + +gevent.spawn(dbCleanup) +gevent.spawn(dbCommitCheck) +atexit.register(dbCloseAll) + + +class DbTableError(Exception): + def __init__(self, message, table): + super().__init__(message) + self.table = table + + +class Db(object): + + def __init__(self, schema, db_path, close_idle=False): + global next_db_id + self.db_path = db_path + self.db_dir = os.path.dirname(db_path) + "/" + self.schema = schema + self.schema["version"] = self.schema.get("version", 1) + self.conn = None + self.cur = None + self.cursors = weakref.WeakSet() + self.id = next_db_id + next_db_id += 1 + self.progress_sleeping = False + self.commiting = False + self.log = logging.getLogger("Db#%s:%s" % (self.id, schema["db_name"])) + self.table_names = None + self.collect_stats = False + self.foreign_keys = False + self.need_commit = False + self.query_stats = {} + self.db_keyvalues = {} + self.delayed_queue = [] + self.delayed_queue_thread = None + self.close_idle = close_idle + self.last_query_time = time.time() + self.last_sleep_time = time.time() + self.num_execute_since_sleep = 0 + self.lock = ThreadPool.Lock() + self.connect_lock = ThreadPool.Lock() + + def __repr__(self): + return "" % (id(self), self.db_path, self.close_idle) + + def connect(self): + self.connect_lock.acquire(True) + try: + if self.conn: + self.log.debug("Already connected, connection ignored") + return + + if self not in opened_dbs: + opened_dbs.append(self) + s = time.time() + try: # Directory not exist yet + os.makedirs(self.db_dir) + self.log.debug("Created Db path: %s" % self.db_dir) + except OSError as err: + if err.errno != errno.EEXIST: + raise err + if not os.path.isfile(self.db_path): + self.log.debug("Db file not exist yet: %s" % self.db_path) + self.conn = sqlite3.connect(self.db_path, isolation_level="DEFERRED", check_same_thread=False) + self.conn.row_factory = sqlite3.Row + self.conn.set_progress_handler(self.progress, 5000000) + self.conn.execute('PRAGMA journal_mode=WAL') + if self.foreign_keys: + self.conn.execute("PRAGMA foreign_keys = ON") + self.cur = self.getCursor() + + self.log.debug( + "Connected to %s in %.3fs (opened: %s, sqlite version: %s)..." % + (self.db_path, time.time() - s, len(opened_dbs), sqlite3.version) + ) + self.log.debug("Connect by thread: %s" % threading.current_thread().ident) + self.log.debug("Connect called by %s" % Debug.formatStack()) + finally: + self.connect_lock.release() + + def getConn(self): + if not self.conn: + self.connect() + return self.conn + + def progress(self, *args, **kwargs): + self.progress_sleeping = True + time.sleep(0.001) + self.progress_sleeping = False + + # Execute query using dbcursor + def execute(self, query, params=None): + if not self.conn: + self.connect() + return self.cur.execute(query, params) + + @thread_pool_db.wrap + def commit(self, reason="Unknown"): + if self.progress_sleeping: + self.log.debug("Commit ignored: Progress sleeping") + return False + + if not self.conn: + self.log.debug("Commit ignored: No connection") + return False + + if self.commiting: + self.log.debug("Commit ignored: Already commiting") + return False + + try: + s = time.time() + self.commiting = True + self.conn.commit() + self.log.debug("Commited in %.3fs (reason: %s)" % (time.time() - s, reason)) + return True + except Exception as err: + if "SQL statements in progress" in str(err): + self.log.warning("Commit delayed: %s (reason: %s)" % (Debug.formatException(err), reason)) + else: + self.log.error("Commit error: %s (reason: %s)" % (Debug.formatException(err), reason)) + return False + finally: + self.commiting = False + + def insertOrUpdate(self, *args, **kwargs): + if not self.conn: + self.connect() + return self.cur.insertOrUpdate(*args, **kwargs) + + def executeDelayed(self, *args, **kwargs): + if not self.delayed_queue_thread: + self.delayed_queue_thread = gevent.spawn_later(1, self.processDelayed) + self.delayed_queue.append(("execute", (args, kwargs))) + + def insertOrUpdateDelayed(self, *args, **kwargs): + if not self.delayed_queue: + gevent.spawn_later(1, self.processDelayed) + self.delayed_queue.append(("insertOrUpdate", (args, kwargs))) + + def processDelayed(self): + if not self.delayed_queue: + self.log.debug("processDelayed aborted") + return + if not self.conn: + self.connect() + + s = time.time() + cur = self.getCursor() + for command, params in self.delayed_queue: + if command == "insertOrUpdate": + cur.insertOrUpdate(*params[0], **params[1]) + else: + cur.execute(*params[0], **params[1]) + + if len(self.delayed_queue) > 10: + self.log.debug("Processed %s delayed queue in %.3fs" % (len(self.delayed_queue), time.time() - s)) + self.delayed_queue = [] + self.delayed_queue_thread = None + + def close(self, reason="Unknown"): + if not self.conn: + return False + self.connect_lock.acquire() + s = time.time() + if self.delayed_queue: + self.processDelayed() + if self in opened_dbs: + opened_dbs.remove(self) + self.need_commit = False + self.commit("Closing: %s" % reason) + self.log.debug("Close called by %s" % Debug.formatStack()) + for i in range(5): + if len(self.cursors) == 0: + break + self.log.debug("Pending cursors: %s" % len(self.cursors)) + time.sleep(0.1 * i) + if len(self.cursors): + self.log.debug("Killing cursors: %s" % len(self.cursors)) + self.conn.interrupt() + + if self.cur: + self.cur.close() + if self.conn: + ThreadPool.main_loop.call(self.conn.close) + self.conn = None + self.cur = None + self.log.debug("%s closed (reason: %s) in %.3fs, opened: %s" % (self.db_path, reason, time.time() - s, len(opened_dbs))) + self.connect_lock.release() + return True + + # Gets a cursor object to database + # Return: Cursor class + def getCursor(self): + if not self.conn: + self.connect() + + cur = DbCursor(self) + return cur + + def getSharedCursor(self): + if not self.conn: + self.connect() + return self.cur + + # Get the table version + # Return: Table version or None if not exist + def getTableVersion(self, table_name): + if not self.db_keyvalues: # Get db keyvalues + try: + res = self.execute("SELECT * FROM keyvalue WHERE json_id=0") # json_id = 0 is internal keyvalues + except sqlite3.OperationalError as err: # Table not exist + self.log.debug("Query table version error: %s" % err) + return False + + for row in res: + self.db_keyvalues[row["key"]] = row["value"] + + return self.db_keyvalues.get("table.%s.version" % table_name, 0) + + # Check Db tables + # Return: Changed table names + def checkTables(self): + s = time.time() + changed_tables = [] + + cur = self.getSharedCursor() + + # Check internal tables + # Check keyvalue table + changed = cur.needTable("keyvalue", [ + ["keyvalue_id", "INTEGER PRIMARY KEY AUTOINCREMENT"], + ["key", "TEXT"], + ["value", "INTEGER"], + ["json_id", "INTEGER"], + ], [ + "CREATE UNIQUE INDEX key_id ON keyvalue(json_id, key)" + ], version=self.schema["version"]) + if changed: + changed_tables.append("keyvalue") + + # Create json table if no custom one defined + if "json" not in self.schema.get("tables", {}): + if self.schema["version"] == 1: + changed = cur.needTable("json", [ + ["json_id", "INTEGER PRIMARY KEY AUTOINCREMENT"], + ["path", "VARCHAR(255)"] + ], [ + "CREATE UNIQUE INDEX path ON json(path)" + ], version=self.schema["version"]) + elif self.schema["version"] == 2: + changed = cur.needTable("json", [ + ["json_id", "INTEGER PRIMARY KEY AUTOINCREMENT"], + ["directory", "VARCHAR(255)"], + ["file_name", "VARCHAR(255)"] + ], [ + "CREATE UNIQUE INDEX path ON json(directory, file_name)" + ], version=self.schema["version"]) + elif self.schema["version"] == 3: + changed = cur.needTable("json", [ + ["json_id", "INTEGER PRIMARY KEY AUTOINCREMENT"], + ["site", "VARCHAR(255)"], + ["directory", "VARCHAR(255)"], + ["file_name", "VARCHAR(255)"] + ], [ + "CREATE UNIQUE INDEX path ON json(directory, site, file_name)" + ], version=self.schema["version"]) + if changed: + changed_tables.append("json") + + # Check schema tables + for table_name, table_settings in self.schema.get("tables", {}).items(): + try: + indexes = table_settings.get("indexes", []) + version = table_settings.get("schema_changed", 0) + changed = cur.needTable( + table_name, table_settings["cols"], + indexes, version=version + ) + if changed: + changed_tables.append(table_name) + except Exception as err: + self.log.error("Error creating table %s: %s" % (table_name, Debug.formatException(err))) + raise DbTableError(err, table_name) + + self.log.debug("Db check done in %.3fs, changed tables: %s" % (time.time() - s, changed_tables)) + if changed_tables: + self.db_keyvalues = {} # Refresh table version cache + + return changed_tables + + # Update json file to db + # Return: True if matched + def updateJson(self, file_path, file=None, cur=None): + if not file_path.startswith(self.db_dir): + return False # Not from the db dir: Skipping + relative_path = file_path[len(self.db_dir):] # File path realative to db file + + # Check if filename matches any of mappings in schema + matched_maps = [] + for match, map_settings in self.schema["maps"].items(): + try: + if SafeRe.match(match, relative_path): + matched_maps.append(map_settings) + except SafeRe.UnsafePatternError as err: + self.log.error(err) + + # No match found for the file + if not matched_maps: + return False + + # Load the json file + try: + if file is None: # Open file is not file object passed + file = open(file_path, "rb") + + if file is False: # File deleted + data = {} + else: + if file_path.endswith("json.gz"): + file = helper.limitedGzipFile(fileobj=file) + + if sys.version_info.major == 3 and sys.version_info.minor < 6: + data = json.loads(file.read().decode("utf8")) + else: + data = json.load(file) + except Exception as err: + self.log.debug("Json file %s load error: %s" % (file_path, err)) + data = {} + + # No cursor specificed + if not cur: + cur = self.getSharedCursor() + cur.logging = False + + # Row for current json file if required + if not data or [dbmap for dbmap in matched_maps if "to_keyvalue" in dbmap or "to_table" in dbmap]: + json_row = cur.getJsonRow(relative_path) + + # Check matched mappings in schema + for dbmap in matched_maps: + # Insert non-relational key values + if dbmap.get("to_keyvalue"): + # Get current values + res = cur.execute("SELECT * FROM keyvalue WHERE json_id = ?", (json_row["json_id"],)) + current_keyvalue = {} + current_keyvalue_id = {} + for row in res: + current_keyvalue[row["key"]] = row["value"] + current_keyvalue_id[row["key"]] = row["keyvalue_id"] + + for key in dbmap["to_keyvalue"]: + if key not in current_keyvalue: # Keyvalue not exist yet in the db + cur.execute( + "INSERT INTO keyvalue ?", + {"key": key, "value": data.get(key), "json_id": json_row["json_id"]} + ) + elif data.get(key) != current_keyvalue[key]: # Keyvalue different value + cur.execute( + "UPDATE keyvalue SET value = ? WHERE keyvalue_id = ?", + (data.get(key), current_keyvalue_id[key]) + ) + + # Insert data to json table for easier joins + if dbmap.get("to_json_table"): + directory, file_name = re.match("^(.*?)/*([^/]*)$", relative_path).groups() + data_json_row = dict(cur.getJsonRow(directory + "/" + dbmap.get("file_name", file_name))) + changed = False + for key in dbmap["to_json_table"]: + if data.get(key) != data_json_row.get(key): + changed = True + if changed: + # Add the custom col values + data_json_row.update({key: val for key, val in data.items() if key in dbmap["to_json_table"]}) + cur.execute("INSERT OR REPLACE INTO json ?", data_json_row) + + # Insert data to tables + for table_settings in dbmap.get("to_table", []): + if isinstance(table_settings, dict): # Custom settings + table_name = table_settings["table"] # Table name to insert datas + node = table_settings.get("node", table_name) # Node keyname in data json file + key_col = table_settings.get("key_col") # Map dict key as this col + val_col = table_settings.get("val_col") # Map dict value as this col + import_cols = table_settings.get("import_cols") + replaces = table_settings.get("replaces") + else: # Simple settings + table_name = table_settings + node = table_settings + key_col = None + val_col = None + import_cols = None + replaces = None + + # Fill import cols from table cols + if not import_cols: + import_cols = set([item[0] for item in self.schema["tables"][table_name]["cols"]]) + + cur.execute("DELETE FROM %s WHERE json_id = ?" % table_name, (json_row["json_id"],)) + + if node not in data: + continue + + if key_col: # Map as dict + for key, val in data[node].items(): + if val_col: # Single value + cur.execute( + "INSERT OR REPLACE INTO %s ?" % table_name, + {key_col: key, val_col: val, "json_id": json_row["json_id"]} + ) + else: # Multi value + if type(val) is dict: # Single row + row = val + if import_cols: + row = {key: row[key] for key in row if key in import_cols} # Filter row by import_cols + row[key_col] = key + # Replace in value if necessary + if replaces: + for replace_key, replace in replaces.items(): + if replace_key in row: + for replace_from, replace_to in replace.items(): + row[replace_key] = row[replace_key].replace(replace_from, replace_to) + + row["json_id"] = json_row["json_id"] + cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row) + elif type(val) is list: # Multi row + for row in val: + row[key_col] = key + row["json_id"] = json_row["json_id"] + cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row) + else: # Map as list + for row in data[node]: + row["json_id"] = json_row["json_id"] + if import_cols: + row = {key: row[key] for key in row if key in import_cols} # Filter row by import_cols + cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row) + + # Cleanup json row + if not data: + self.log.debug("Cleanup json row for %s" % file_path) + cur.execute("DELETE FROM json WHERE json_id = %s" % json_row["json_id"]) + + return True + + +if __name__ == "__main__": + s = time.time() + console_log = logging.StreamHandler() + logging.getLogger('').setLevel(logging.DEBUG) + logging.getLogger('').addHandler(console_log) + console_log.setLevel(logging.DEBUG) + dbjson = Db(json.load(open("zerotalk.schema.json")), "data/users/zerotalk.db") + dbjson.collect_stats = True + dbjson.checkTables() + cur = dbjson.getCursor() + cur.logging = False + dbjson.updateJson("data/users/content.json", cur=cur) + for user_dir in os.listdir("data/users"): + if os.path.isdir("data/users/%s" % user_dir): + dbjson.updateJson("data/users/%s/data.json" % user_dir, cur=cur) + # print ".", + cur.logging = True + print("Done in %.3fs" % (time.time() - s)) + for query, stats in sorted(dbjson.query_stats.items()): + print("-", query, stats) diff --git a/src/Db/DbCursor.py b/src/Db/DbCursor.py new file mode 100644 index 00000000..acb8846d --- /dev/null +++ b/src/Db/DbCursor.py @@ -0,0 +1,246 @@ +import time +import re +from util import helper + +# Special sqlite cursor + + +class DbCursor: + + def __init__(self, db): + self.db = db + self.logging = False + + def quoteValue(self, value): + if type(value) is int: + return str(value) + else: + return "'%s'" % value.replace("'", "''") + + def parseQuery(self, query, params): + query_type = query.split(" ", 1)[0].upper() + if isinstance(params, dict) and "?" in query: # Make easier select and insert by allowing dict params + if query_type in ("SELECT", "DELETE", "UPDATE"): + # Convert param dict to SELECT * FROM table WHERE key = ? AND key2 = ? format + query_wheres = [] + values = [] + for key, value in params.items(): + if type(value) is list: + if key.startswith("not__"): + field = key.replace("not__", "") + operator = "NOT IN" + else: + field = key + operator = "IN" + if len(value) > 100: + # Embed values in query to avoid "too many SQL variables" error + query_values = ",".join(map(helper.sqlquote, value)) + else: + query_values = ",".join(["?"] * len(value)) + values += value + query_wheres.append( + "%s %s (%s)" % + (field, operator, query_values) + ) + else: + if key.startswith("not__"): + query_wheres.append(key.replace("not__", "") + " != ?") + elif key.endswith("__like"): + query_wheres.append(key.replace("__like", "") + " LIKE ?") + elif key.endswith(">"): + query_wheres.append(key.replace(">", "") + " > ?") + elif key.endswith("<"): + query_wheres.append(key.replace("<", "") + " < ?") + else: + query_wheres.append(key + " = ?") + values.append(value) + wheres = " AND ".join(query_wheres) + if wheres == "": + wheres = "1" + query = re.sub("(.*)[?]", "\\1 %s" % wheres, query) # Replace the last ? + params = values + else: + # Convert param dict to INSERT INTO table (key, key2) VALUES (?, ?) format + keys = ", ".join(params.keys()) + values = ", ".join(['?' for key in params.keys()]) + keysvalues = "(%s) VALUES (%s)" % (keys, values) + query = re.sub("(.*)[?]", "\\1%s" % keysvalues, query) # Replace the last ? + params = tuple(params.values()) + elif isinstance(params, dict) and ":" in query: + new_params = dict() + values = [] + for key, value in params.items(): + if type(value) is list: + for idx, val in enumerate(value): + new_params[key + "__" + str(idx)] = val + + new_names = [":" + key + "__" + str(idx) for idx in range(len(value))] + query = re.sub(r":" + re.escape(key) + r"([)\s]|$)", "(%s)%s" % (", ".join(new_names), r"\1"), query) + else: + new_params[key] = value + + params = new_params + return query, params + + def execute(self, query, params=None): + query = query.strip() + while self.db.progress_sleeping or self.db.commiting: + time.sleep(0.1) + + self.db.last_query_time = time.time() + + query, params = self.parseQuery(query, params) + + cursor = self.db.getConn().cursor() + self.db.cursors.add(cursor) + if self.db.lock.locked(): + self.db.log.debug("Locked for %.3fs" % (time.time() - self.db.lock.time_lock)) + + try: + s = time.time() + self.db.lock.acquire(True) + if query.upper().strip("; ") == "VACUUM": + self.db.commit("vacuum called") + if params: + res = cursor.execute(query, params) + else: + res = cursor.execute(query) + finally: + self.db.lock.release() + + taken_query = time.time() - s + if self.logging or taken_query > 1: + if params: # Query has parameters + self.db.log.debug("Query: " + query + " " + str(params) + " (Done in %.4f)" % (time.time() - s)) + else: + self.db.log.debug("Query: " + query + " (Done in %.4f)" % (time.time() - s)) + + # Log query stats + if self.db.collect_stats: + if query not in self.db.query_stats: + self.db.query_stats[query] = {"call": 0, "time": 0.0} + self.db.query_stats[query]["call"] += 1 + self.db.query_stats[query]["time"] += time.time() - s + + query_type = query.split(" ", 1)[0].upper() + is_update_query = query_type in ["UPDATE", "DELETE", "INSERT", "CREATE"] + if not self.db.need_commit and is_update_query: + self.db.need_commit = True + + if is_update_query: + return cursor + else: + return res + + def executemany(self, query, params): + while self.db.progress_sleeping or self.db.commiting: + time.sleep(0.1) + + self.db.last_query_time = time.time() + + s = time.time() + cursor = self.db.getConn().cursor() + self.db.cursors.add(cursor) + + try: + self.db.lock.acquire(True) + cursor.executemany(query, params) + finally: + self.db.lock.release() + + taken_query = time.time() - s + if self.logging or taken_query > 0.1: + self.db.log.debug("Execute many: %s (Done in %.4f)" % (query, taken_query)) + + self.db.need_commit = True + + return cursor + + # Creates on updates a database row without incrementing the rowid + def insertOrUpdate(self, table, query_sets, query_wheres, oninsert={}): + sql_sets = ["%s = :%s" % (key, key) for key in query_sets.keys()] + sql_wheres = ["%s = :%s" % (key, key) for key in query_wheres.keys()] + + params = query_sets + params.update(query_wheres) + res = self.execute( + "UPDATE %s SET %s WHERE %s" % (table, ", ".join(sql_sets), " AND ".join(sql_wheres)), + params + ) + if res.rowcount == 0: + params.update(oninsert) # Add insert-only fields + self.execute("INSERT INTO %s ?" % table, params) + + # Create new table + # Return: True on success + def createTable(self, table, cols): + # TODO: Check current structure + self.execute("DROP TABLE IF EXISTS %s" % table) + col_definitions = [] + for col_name, col_type in cols: + col_definitions.append("%s %s" % (col_name, col_type)) + + self.execute("CREATE TABLE %s (%s)" % (table, ",".join(col_definitions))) + return True + + # Create indexes on table + # Return: True on success + def createIndexes(self, table, indexes): + for index in indexes: + if not index.strip().upper().startswith("CREATE"): + self.db.log.error("Index command should start with CREATE: %s" % index) + continue + self.execute(index) + + # Create table if not exist + # Return: True if updated + def needTable(self, table, cols, indexes=None, version=1): + current_version = self.db.getTableVersion(table) + if int(current_version) < int(version): # Table need update or not extis + self.db.log.debug("Table %s outdated...version: %s need: %s, rebuilding..." % (table, current_version, version)) + self.createTable(table, cols) + if indexes: + self.createIndexes(table, indexes) + self.execute( + "INSERT OR REPLACE INTO keyvalue ?", + {"json_id": 0, "key": "table.%s.version" % table, "value": version} + ) + return True + else: # Not changed + return False + + # Get or create a row for json file + # Return: The database row + def getJsonRow(self, file_path): + directory, file_name = re.match("^(.*?)/*([^/]*)$", file_path).groups() + if self.db.schema["version"] == 1: + # One path field + res = self.execute("SELECT * FROM json WHERE ? LIMIT 1", {"path": file_path}) + row = res.fetchone() + if not row: # No row yet, create it + self.execute("INSERT INTO json ?", {"path": file_path}) + res = self.execute("SELECT * FROM json WHERE ? LIMIT 1", {"path": file_path}) + row = res.fetchone() + elif self.db.schema["version"] == 2: + # Separate directory, file_name (easier join) + res = self.execute("SELECT * FROM json WHERE ? LIMIT 1", {"directory": directory, "file_name": file_name}) + row = res.fetchone() + if not row: # No row yet, create it + self.execute("INSERT INTO json ?", {"directory": directory, "file_name": file_name}) + res = self.execute("SELECT * FROM json WHERE ? LIMIT 1", {"directory": directory, "file_name": file_name}) + row = res.fetchone() + elif self.db.schema["version"] == 3: + # Separate site, directory, file_name (for merger sites) + site_address, directory = re.match("^([^/]*)/(.*)$", directory).groups() + res = self.execute("SELECT * FROM json WHERE ? LIMIT 1", {"site": site_address, "directory": directory, "file_name": file_name}) + row = res.fetchone() + if not row: # No row yet, create it + self.execute("INSERT INTO json ?", {"site": site_address, "directory": directory, "file_name": file_name}) + res = self.execute("SELECT * FROM json WHERE ? LIMIT 1", {"site": site_address, "directory": directory, "file_name": file_name}) + row = res.fetchone() + else: + raise Exception("Dbschema version %s not supported" % self.db.schema.get("version")) + return row + + def close(self): + pass diff --git a/src/Db/DbQuery.py b/src/Db/DbQuery.py new file mode 100644 index 00000000..3fb5ef73 --- /dev/null +++ b/src/Db/DbQuery.py @@ -0,0 +1,46 @@ +import re + + +# Parse and modify sql queries +class DbQuery: + def __init__(self, query): + self.setQuery(query.strip()) + + # Split main parts of query + def parseParts(self, query): + parts = re.split("(SELECT|FROM|WHERE|ORDER BY|LIMIT)", query) + parts = [_f for _f in parts if _f] # Remove empty parts + parts = [s.strip() for s in parts] # Remove whitespace + return dict(list(zip(parts[0::2], parts[1::2]))) + + # Parse selected fields SELECT ... FROM + def parseFields(self, query_select): + fields = re.findall("([^,]+) AS ([^,]+)", query_select) + return {key: val.strip() for val, key in fields} + + # Parse query conditions WHERE ... + def parseWheres(self, query_where): + if " AND " in query_where: + return query_where.split(" AND ") + elif query_where: + return [query_where] + else: + return [] + + # Set the query + def setQuery(self, query): + self.parts = self.parseParts(query) + self.fields = self.parseFields(self.parts["SELECT"]) + self.wheres = self.parseWheres(self.parts.get("WHERE", "")) + + # Convert query back to string + def __str__(self): + query_parts = [] + for part_name in ["SELECT", "FROM", "WHERE", "ORDER BY", "LIMIT"]: + if part_name == "WHERE" and self.wheres: + query_parts.append("WHERE") + query_parts.append(" AND ".join(self.wheres)) + elif part_name in self.parts: + query_parts.append(part_name) + query_parts.append(self.parts[part_name]) + return "\n".join(query_parts) diff --git a/src/Db/__init__.py b/src/Db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Debug/Debug.py b/src/Debug/Debug.py new file mode 100644 index 00000000..0ec42615 --- /dev/null +++ b/src/Debug/Debug.py @@ -0,0 +1,186 @@ +import sys +import os +import re +from Config import config + + +# Non fatal exception +class Notify(Exception): + def __init__(self, message=None): + if message: + self.message = message + + def __str__(self): + return self.message + + +# Gevent greenlet.kill accept Exception type +def createNotifyType(message): + return type("Notify", (Notify, ), {"message": message}) + + +def formatExceptionMessage(err): + err_type = err.__class__.__name__ + if err.args: + err_message = err.args[-1] + else: + err_message = err.__str__() + return "%s: %s" % (err_type, err_message) + + +python_lib_dirs = [path.replace("\\", "/") for path in sys.path if re.sub(r".*[\\/]", "", path) in ("site-packages", "dist-packages")] +python_lib_dirs.append(os.path.dirname(os.__file__).replace("\\", "/")) # TODO: check if returns the correct path for PyPy + +root_dir = os.path.realpath(os.path.dirname(__file__) + "/../../") +root_dir = root_dir.replace("\\", "/") + + +def formatTraceback(items, limit=None, fold_builtin=True): + back = [] + i = 0 + prev_file_title = "" + is_prev_builtin = False + + for path, line in items: + i += 1 + is_last = i == len(items) + path = path.replace("\\", "/") + + if path.startswith("src/gevent/"): + file_title = "/" + path[len("src/gevent/"):] + is_builtin = True + is_skippable_builtin = False + elif path in ("", ""): + file_title = "(importlib)" + is_builtin = True + is_skippable_builtin = True + else: + is_skippable_builtin = False + for base in python_lib_dirs: + if path.startswith(base + "/"): + file_title = path[len(base + "/"):] + module_name, *tail = file_title.split("/") + if module_name.endswith(".py"): + module_name = module_name[:-3] + file_title = "/".join(["<%s>" % module_name] + tail) + is_builtin = True + break + else: + is_builtin = False + for base in (root_dir + "/src", root_dir + "/plugins", root_dir): + if path.startswith(base + "/"): + file_title = path[len(base + "/"):] + break + else: + # For unknown paths, do our best to hide absolute path + file_title = path + for needle in ("/zeronet/", "/core/"): + if needle in file_title.lower(): + file_title = "?/" + file_title[file_title.lower().rindex(needle) + len(needle):] + + # Path compression: A/AB/ABC/X/Y.py -> ABC/X/Y.py + # E.g.: in 'Db/DbCursor.py' the directory part is unnecessary + if not file_title.startswith("/"): + prev_part = "" + for i, part in enumerate(file_title.split("/") + [""]): + if not part.startswith(prev_part): + break + prev_part = part + file_title = "/".join(file_title.split("/")[i - 1:]) + + if is_skippable_builtin and fold_builtin: + pass + elif is_builtin and is_prev_builtin and not is_last and fold_builtin: + if back[-1] != "...": + back.append("...") + else: + if file_title == prev_file_title: + back.append("%s" % line) + else: + back.append("%s line %s" % (file_title, line)) + + prev_file_title = file_title + is_prev_builtin = is_builtin + + if limit and i >= limit: + back.append("...") + break + return back + + +def formatException(err=None, format="text"): + import traceback + if type(err) == Notify: + return err + elif type(err) == tuple and err and err[0] is not None: # Passed trackeback info + exc_type, exc_obj, exc_tb = err + err = None + else: # No trackeback info passed, get latest + exc_type, exc_obj, exc_tb = sys.exc_info() + + if not err: + if hasattr(err, "message"): + err = exc_obj.message + else: + err = exc_obj + + tb = formatTraceback([[frame[0], frame[1]] for frame in traceback.extract_tb(exc_tb)]) + if format == "html": + return "%s: %s
%s" % (repr(err), err, " > ".join(tb)) + else: + return "%s: %s in %s" % (exc_type.__name__, err, " > ".join(tb)) + + +def formatStack(limit=None): + import inspect + tb = formatTraceback([[frame[1], frame[2]] for frame in inspect.stack()[1:]], limit=limit) + return " > ".join(tb) + + +# Test if gevent eventloop blocks +import logging +import gevent +import time + + +num_block = 0 + + +def testBlock(): + global num_block + logging.debug("Gevent block checker started") + last_time = time.time() + while 1: + time.sleep(1) + if time.time() - last_time > 1.1: + logging.debug("Gevent block detected: %.3fs" % (time.time() - last_time - 1)) + num_block += 1 + last_time = time.time() + + +gevent.spawn(testBlock) + + +if __name__ == "__main__": + try: + print(1 / 0) + except Exception as err: + print(type(err).__name__) + print("1/0 error: %s" % formatException(err)) + + def loadJson(): + json.loads("Errr") + + import json + try: + loadJson() + except Exception as err: + print(err) + print("Json load error: %s" % formatException(err)) + + try: + raise Notify("nothing...") + except Exception as err: + print("Notify: %s" % formatException(err)) + + loadJson() diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py new file mode 100644 index 00000000..d100a3b8 --- /dev/null +++ b/src/Debug/DebugHook.py @@ -0,0 +1,115 @@ +import sys +import logging +import signal +import importlib + +import gevent +import gevent.hub + +from Config import config +from . import Debug + +last_error = None + +def shutdown(reason="Unknown"): + logging.info("Shutting down (reason: %s)..." % reason) + import main + if "file_server" in dir(main): + try: + gevent.spawn(main.file_server.stop) + if "ui_server" in dir(main): + gevent.spawn(main.ui_server.stop) + except Exception as err: + print("Proper shutdown error: %s" % err) + sys.exit(0) + else: + sys.exit(0) + +# Store last error, ignore notify, allow manual error logging +def handleError(*args, **kwargs): + global last_error + if not args: # Manual called + args = sys.exc_info() + silent = True + else: + silent = False + if args[0].__name__ != "Notify": + last_error = args + + if args[0].__name__ == "KeyboardInterrupt": + shutdown("Keyboard interrupt") + elif not silent and args[0].__name__ != "Notify": + logging.exception("Unhandled exception") + if "greenlet.py" not in args[2].tb_frame.f_code.co_filename: # Don't display error twice + sys.__excepthook__(*args, **kwargs) + + +# Ignore notify errors +def handleErrorNotify(*args, **kwargs): + err = args[0] + if err.__name__ == "KeyboardInterrupt": + shutdown("Keyboard interrupt") + elif err.__name__ != "Notify": + logging.error("Unhandled exception: %s" % Debug.formatException(args)) + sys.__excepthook__(*args, **kwargs) + + +if config.debug: # Keep last error for /Debug + sys.excepthook = handleError +else: + sys.excepthook = handleErrorNotify + + +# Override default error handler to allow silent killing / custom logging +if "handle_error" in dir(gevent.hub.Hub): + gevent.hub.Hub._original_handle_error = gevent.hub.Hub.handle_error +else: + logging.debug("gevent.hub.Hub.handle_error not found using old gevent hooks") + OriginalGreenlet = gevent.Greenlet + class ErrorhookedGreenlet(OriginalGreenlet): + def _report_error(self, exc_info): + sys.excepthook(exc_info[0], exc_info[1], exc_info[2]) + + gevent.Greenlet = gevent.greenlet.Greenlet = ErrorhookedGreenlet + importlib.reload(gevent) + +def handleGreenletError(context, type, value, tb): + if context.__class__ is tuple and context[0].__class__.__name__ == "ThreadPool": + # Exceptions in ThreadPool will be handled in the main Thread + return None + + if isinstance(value, str): + # Cython can raise errors where the value is a plain string + # e.g., AttributeError, "_semaphore.Semaphore has no attr", + value = type(value) + + if not issubclass(type, gevent.get_hub().NOT_ERROR): + sys.excepthook(type, value, tb) + +gevent.get_hub().handle_error = handleGreenletError + +try: + signal.signal(signal.SIGTERM, lambda signum, stack_frame: shutdown("SIGTERM")) +except Exception as err: + logging.debug("Error setting up SIGTERM watcher: %s" % err) + + +if __name__ == "__main__": + import time + from gevent import monkey + monkey.patch_all(thread=False, ssl=False) + from . import Debug + + def sleeper(num): + print("started", num) + time.sleep(3) + raise Exception("Error") + print("stopped", num) + thread1 = gevent.spawn(sleeper, 1) + thread2 = gevent.spawn(sleeper, 2) + time.sleep(1) + print("killing...") + thread1.kill(exception=Debug.Notify("Worker stopped")) + #thread2.throw(Debug.Notify("Throw")) + print("killed") + gevent.joinall([thread1,thread2]) diff --git a/src/Debug/DebugLock.py b/src/Debug/DebugLock.py new file mode 100644 index 00000000..9cf22520 --- /dev/null +++ b/src/Debug/DebugLock.py @@ -0,0 +1,24 @@ +import time +import logging + +import gevent.lock + +from Debug import Debug + + +class DebugLock: + def __init__(self, log_after=0.01, name="Lock"): + self.name = name + self.log_after = log_after + self.lock = gevent.lock.Semaphore(1) + self.release = self.lock.release + + def acquire(self, *args, **kwargs): + s = time.time() + res = self.lock.acquire(*args, **kwargs) + time_taken = time.time() - s + if time_taken >= self.log_after: + logging.debug("%s: Waited %.3fs after called by %s" % + (self.name, time_taken, Debug.formatStack()) + ) + return res diff --git a/src/Debug/DebugMedia.py b/src/Debug/DebugMedia.py new file mode 100644 index 00000000..a892dc56 --- /dev/null +++ b/src/Debug/DebugMedia.py @@ -0,0 +1,135 @@ +import os +import subprocess +import re +import logging +import time +import functools + +from Config import config +from util import helper + + +# Find files with extension in path +def findfiles(path, find_ext): + def sorter(f1, f2): + f1 = f1[0].replace(path, "") + f2 = f2[0].replace(path, "") + if f1 == "": + return 1 + elif f2 == "": + return -1 + else: + return helper.cmp(f1.lower(), f2.lower()) + + for root, dirs, files in sorted(os.walk(path, topdown=False), key=functools.cmp_to_key(sorter)): + for file in sorted(files): + file_path = root + "/" + file + file_ext = file.split(".")[-1] + if file_ext in find_ext and not file.startswith("all."): + yield file_path.replace("\\", "/") + + +# Try to find coffeescript compiler in path +def findCoffeescriptCompiler(): + coffeescript_compiler = None + try: + import distutils.spawn + coffeescript_compiler = helper.shellquote(distutils.spawn.find_executable("coffee")) + " --no-header -p" + except: + pass + if coffeescript_compiler: + return coffeescript_compiler + else: + return False + + +# Generates: all.js: merge *.js, compile coffeescript, all.css: merge *.css, vendor prefix features +def merge(merged_path): + merged_path = merged_path.replace("\\", "/") + merge_dir = os.path.dirname(merged_path) + s = time.time() + ext = merged_path.split(".")[-1] + if ext == "js": # If merging .js find .coffee too + find_ext = ["js", "coffee"] + else: + find_ext = [ext] + + # If exist check the other files modification date + if os.path.isfile(merged_path): + merged_mtime = os.path.getmtime(merged_path) + else: + merged_mtime = 0 + + changed = {} + for file_path in findfiles(merge_dir, find_ext): + if os.path.getmtime(file_path) > merged_mtime + 1: + changed[file_path] = True + if not changed: + return # Assets not changed, nothing to do + + old_parts = {} + if os.path.isfile(merged_path): # Find old parts to avoid unncessary recompile + merged_old = open(merged_path, "rb").read() + for match in re.findall(rb"(/\* ---- (.*?) ---- \*/(.*?)(?=/\* ----|$))", merged_old, re.DOTALL): + old_parts[match[1].decode()] = match[2].strip(b"\n\r") + + logging.debug("Merging %s (changed: %s, old parts: %s)" % (merged_path, changed, len(old_parts))) + # Merge files + parts = [] + s_total = time.time() + for file_path in findfiles(merge_dir, find_ext): + file_relative_path = file_path.replace(merge_dir + "/", "") + parts.append(b"\n/* ---- %s ---- */\n\n" % file_relative_path.encode("utf8")) + if file_path.endswith(".coffee"): # Compile coffee script + if file_path in changed or file_relative_path not in old_parts: # Only recompile if changed or its not compiled before + if config.coffeescript_compiler is None: + config.coffeescript_compiler = findCoffeescriptCompiler() + if not config.coffeescript_compiler: + logging.error("No coffeescript compiler defined, skipping compiling %s" % merged_path) + return False # No coffeescript compiler, skip this file + + # Replace / with os separators and escape it + file_path_escaped = helper.shellquote(file_path.replace("/", os.path.sep)) + + if "%s" in config.coffeescript_compiler: # Replace %s with coffeescript file + command = config.coffeescript_compiler.replace("%s", file_path_escaped) + else: # Put coffeescript file to end + command = config.coffeescript_compiler + " " + file_path_escaped + + # Start compiling + s = time.time() + compiler = subprocess.Popen(command, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + out = compiler.stdout.read() + compiler.wait() + logging.debug("Running: %s (Done in %.2fs)" % (command, time.time() - s)) + + # Check errors + if out and out.startswith(b"("): # No error found + parts.append(out) + else: # Put error message in place of source code + error = out + logging.error("%s Compile error: %s" % (file_relative_path, error)) + error_escaped = re.escape(error).replace(b"\n", b"\\n").replace(br"\\n", br"\n") + parts.append( + b"alert('%s compile error: %s');" % + (file_relative_path.encode(), error_escaped) + ) + else: # Not changed use the old_part + parts.append(old_parts[file_relative_path]) + else: # Add to parts + parts.append(open(file_path, "rb").read()) + + merged = b"\n".join(parts) + if ext == "css": # Vendor prefix css + from lib.cssvendor import cssvendor + merged = cssvendor.prefix(merged) + merged = merged.replace(b"\r", b"") + open(merged_path, "wb").write(merged) + logging.debug("Merged %s (%.2fs)" % (merged_path, time.time() - s_total)) + + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.DEBUG) + os.chdir("..") + config.coffeescript_compiler = r'type "%s" | tools\coffee-node\bin\node.exe tools\coffee-node\bin\coffee --no-header -s -p' + merge("data/12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH/js/all.js") diff --git a/src/Debug/DebugReloader.py b/src/Debug/DebugReloader.py new file mode 100644 index 00000000..482c7921 --- /dev/null +++ b/src/Debug/DebugReloader.py @@ -0,0 +1,69 @@ +import logging +import time +import os + +from Config import config + +if config.debug and config.action == "main": + try: + import watchdog + import watchdog.observers + import watchdog.events + logging.debug("Watchdog fs listener detected, source code autoreload enabled") + enabled = True + except Exception as err: + logging.debug("Watchdog fs listener could not be loaded: %s" % err) + enabled = False +else: + enabled = False + + +class DebugReloader: + def __init__(self, paths=None): + if not paths: + paths = ["src", "plugins", config.data_dir + "/__plugins__"] + self.log = logging.getLogger("DebugReloader") + self.last_chaged = 0 + self.callbacks = [] + if enabled: + self.observer = watchdog.observers.Observer() + event_handler = watchdog.events.FileSystemEventHandler() + event_handler.on_modified = event_handler.on_deleted = self.onChanged + event_handler.on_created = event_handler.on_moved = self.onChanged + for path in paths: + if not os.path.isdir(path): + continue + self.log.debug("Adding autoreload: %s" % path) + self.observer.schedule(event_handler, path, recursive=True) + self.observer.start() + + def addCallback(self, f): + self.callbacks.append(f) + + def onChanged(self, evt): + path = evt.src_path + ext = path.rsplit(".", 1)[-1] + if ext not in ["py", "json"] or "Test" in path or time.time() - self.last_chaged < 1.0: + return False + self.last_chaged = time.time() + if os.path.isfile(path): + time_modified = os.path.getmtime(path) + else: + time_modified = 0 + self.log.debug("File changed: %s reloading source code (modified %.3fs ago)" % (evt, time.time() - time_modified)) + if time.time() - time_modified > 5: # Probably it's just an attribute change, ignore it + return False + + time.sleep(0.1) # Wait for lock release + for callback in self.callbacks: + try: + callback() + except Exception as err: + self.log.exception(err) + + def stop(self): + if enabled: + self.observer.stop() + self.log.debug("Stopped autoreload observer") + +watcher = DebugReloader() diff --git a/src/Debug/__init__.py b/src/Debug/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py new file mode 100644 index 00000000..c082c378 --- /dev/null +++ b/src/File/FileRequest.py @@ -0,0 +1,450 @@ +# Included modules +import os +import time +import json +import collections +import itertools + +# Third party modules +import gevent + +from Debug import Debug +from Config import config +from util import RateLimit +from util import Msgpack +from util import helper +from Plugin import PluginManager +from contextlib import closing + +FILE_BUFF = 1024 * 512 + + +class RequestError(Exception): + pass + + +# Incoming requests +@PluginManager.acceptPlugins +class FileRequest(object): + __slots__ = ("server", "connection", "req_id", "sites", "log", "responded") + + def __init__(self, server, connection): + self.server = server + self.connection = connection + + self.req_id = None + self.sites = self.server.sites + self.log = server.log + self.responded = False # Responded to the request + + def send(self, msg, streaming=False): + if not self.connection.closed: + self.connection.send(msg, streaming) + + def sendRawfile(self, file, read_bytes): + if not self.connection.closed: + self.connection.sendRawfile(file, read_bytes) + + def response(self, msg, streaming=False): + if self.responded: + if config.verbose: + self.log.debug("Req id %s already responded" % self.req_id) + return + if not isinstance(msg, dict): # If msg not a dict create a {"body": msg} + msg = {"body": msg} + msg["cmd"] = "response" + msg["to"] = self.req_id + self.responded = True + self.send(msg, streaming=streaming) + + # Route file requests + def route(self, cmd, req_id, params): + self.req_id = req_id + # Don't allow other sites than locked + if "site" in params and self.connection.target_onion: + valid_sites = self.connection.getValidSites() + if params["site"] not in valid_sites and valid_sites != ["global"]: + self.response({"error": "Invalid site"}) + self.connection.log( + "Site lock violation: %s not in %s, target onion: %s" % + (params["site"], valid_sites, self.connection.target_onion) + ) + self.connection.badAction(5) + return False + + if cmd == "update": + event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"]) + # If called more than once within 15 sec only keep the last update + RateLimit.callAsync(event, max(self.connection.bad_actions, 15), self.actionUpdate, params) + else: + func_name = "action" + cmd[0].upper() + cmd[1:] + func = getattr(self, func_name, None) + if cmd not in ["getFile", "streamFile"]: # Skip IO bound functions + if self.connection.cpu_time > 0.5: + self.log.debug( + "Delay %s %s, cpu_time used by connection: %.3fs" % + (self.connection.ip, cmd, self.connection.cpu_time) + ) + time.sleep(self.connection.cpu_time) + if self.connection.cpu_time > 5: + self.connection.close("Cpu time: %.3fs" % self.connection.cpu_time) + s = time.time() + if func: + func(params) + else: + self.actionUnknown(cmd, params) + + if cmd not in ["getFile", "streamFile"]: + taken = time.time() - s + taken_sent = self.connection.last_sent_time - self.connection.last_send_time + self.connection.cpu_time += taken - taken_sent + + # Update a site file request + def actionUpdate(self, params): + site = self.sites.get(params["site"]) + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(1) + self.connection.badAction(5) + return False + + inner_path = params.get("inner_path", "") + if not inner_path.endswith("content.json"): + self.response({"error": "Only content.json update allowed"}) + self.connection.badAction(5) + return + + current_content_modified = site.content_manager.contents.get(inner_path, {}).get("modified", 0) + should_validate_content = True + if "modified" in params and params["modified"] <= current_content_modified: + should_validate_content = False + valid = None # Same or earlier content as we have + + body = params["body"] + if not body: # No body sent, we have to download it first + site.log.debug("Missing body from update for file %s, downloading ..." % inner_path) + peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer + try: + body = peer.getFile(site.address, inner_path).read() + except Exception as err: + site.log.debug("Can't download updated file %s: %s" % (inner_path, err)) + self.response({"error": "Invalid File update: Failed to download updated file content"}) + self.connection.badAction(5) + return + + if should_validate_content: + try: + if type(body) is str: + body = body.encode() + # elif type(body) is list: + # content = json.loads(bytes(list).decode()) + content = json.loads(body.decode()) + except Exception as err: + site.log.debug("Update for %s is invalid JSON: %s" % (inner_path, err)) + self.response({"error": "File invalid JSON"}) + self.connection.badAction(5) + return + + file_uri = "%s/%s:%s" % (site.address, inner_path, content["modified"]) + + if self.server.files_parsing.get(file_uri): # Check if we already working on it + valid = None # Same file + else: + try: + valid = site.content_manager.verifyFile(inner_path, content) + except Exception as err: + site.log.debug("Update for %s is invalid: %s" % (inner_path, err)) + error = err + valid = False + + if valid is True: # Valid and changed + site.log.info("Update for %s looks valid, saving..." % inner_path) + self.server.files_parsing[file_uri] = True + site.storage.write(inner_path, body) + del params["body"] + + site.onFileDone(inner_path) # Trigger filedone + + # Download every changed file from peer + peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer + # On complete publish to other peers + diffs = params.get("diffs", {}) + site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs, limit=6), "publish_%s" % inner_path) + + # Load new content file and download changed files in new thread + def downloader(): + site.downloadContent(inner_path, peer=peer, diffs=params.get("diffs", {})) + del self.server.files_parsing[file_uri] + + gevent.spawn(downloader) + + self.response({"ok": "Thanks, file %s updated!" % inner_path}) + self.connection.goodAction() + + elif valid is None: # Not changed + peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update old") # Add or get peer + if peer: + if not peer.connection: + peer.connect(self.connection) # Assign current connection to peer + if inner_path in site.content_manager.contents: + peer.last_content_json_update = site.content_manager.contents[inner_path]["modified"] + if config.verbose: + site.log.debug( + "Same version, adding new peer for locked files: %s, tasks: %s" % + (peer.key, len(site.worker_manager.tasks)) + ) + for task in site.worker_manager.tasks: # New peer add to every ongoing task + if task["peers"] and not task["optional_hash_id"]: + # Download file from this peer too if its peer locked + site.needFile(task["inner_path"], peer=peer, update=True, blocking=False) + + self.response({"ok": "File not changed"}) + self.connection.badAction() + + else: # Invalid sign or sha hash + self.response({"error": "File %s invalid: %s" % (inner_path, error)}) + self.connection.badAction(5) + + def isReadable(self, site, inner_path, file, pos): + return True + + # Send file content request + def handleGetFile(self, params, streaming=False): + site = self.sites.get(params["site"]) + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(5) + return False + try: + file_path = site.storage.getPath(params["inner_path"]) + if streaming: + file_obj = site.storage.open(params["inner_path"]) + else: + file_obj = Msgpack.FilePart(file_path, "rb") + + with file_obj as file: + file.seek(params["location"]) + read_bytes = params.get("read_bytes", FILE_BUFF) + file_size = os.fstat(file.fileno()).st_size + + if file_size > read_bytes: # Check if file is readable at current position (for big files) + if not self.isReadable(site, params["inner_path"], file, params["location"]): + raise RequestError("File not readable at position: %s" % params["location"]) + else: + if params.get("file_size") and params["file_size"] != file_size: + self.connection.badAction(2) + raise RequestError("File size does not match: %sB != %sB" % (params["file_size"], file_size)) + + if not streaming: + file.read_bytes = read_bytes + + if params["location"] > file_size: + self.connection.badAction(5) + raise RequestError("Bad file location") + + if streaming: + back = { + "size": file_size, + "location": min(file.tell() + read_bytes, file_size), + "stream_bytes": min(read_bytes, file_size - params["location"]) + } + self.response(back) + self.sendRawfile(file, read_bytes=read_bytes) + else: + back = { + "body": file, + "size": file_size, + "location": min(file.tell() + file.read_bytes, file_size) + } + self.response(back, streaming=True) + + bytes_sent = min(read_bytes, file_size - params["location"]) # Number of bytes we going to send + site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent + if config.debug_socket: + self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent)) + + # Add peer to site if not added before + connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") + if connected_peer: # Just added + connected_peer.connect(self.connection) # Assign current connection to peer + + return {"bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"]} + + except RequestError as err: + self.log.debug("GetFile %s %s %s request error: %s" % (self.connection, params["site"], params["inner_path"], Debug.formatException(err))) + self.response({"error": "File read error: %s" % err}) + except OSError as err: + if config.verbose: + self.log.debug("GetFile read error: %s" % Debug.formatException(err)) + self.response({"error": "File read error"}) + return False + except Exception as err: + self.log.error("GetFile exception: %s" % Debug.formatException(err)) + self.response({"error": "File read exception"}) + return False + + def actionGetFile(self, params): + return self.handleGetFile(params) + + def actionStreamFile(self, params): + return self.handleGetFile(params, streaming=True) + + # Peer exchange request + def actionPex(self, params): + site = self.sites.get(params["site"]) + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(5) + return False + + got_peer_keys = [] + added = 0 + + # Add requester peer to site + connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") + + if connected_peer: # It was not registered before + added += 1 + connected_peer.connect(self.connection) # Assign current connection to peer + + # Add sent peers to site + for packed_address in itertools.chain(params.get("peers", []), params.get("peers_ipv6", [])): + address = helper.unpackAddress(packed_address) + got_peer_keys.append("%s:%s" % address) + if site.addPeer(*address, source="pex"): + added += 1 + + # Add sent onion peers to site + for packed_address in params.get("peers_onion", []): + address = helper.unpackOnionAddress(packed_address) + got_peer_keys.append("%s:%s" % address) + if site.addPeer(*address, source="pex"): + added += 1 + + # Send back peers that is not in the sent list and connectable (not port 0) + packed_peers = helper.packPeers(site.getConnectablePeers(params["need"], ignore=got_peer_keys, allow_private=False)) + + if added: + site.worker_manager.onPeers() + if config.verbose: + self.log.debug( + "Added %s peers to %s using pex, sending back %s" % + (added, site, {key: len(val) for key, val in packed_peers.items()}) + ) + + back = { + "peers": packed_peers["ipv4"], + "peers_ipv6": packed_peers["ipv6"], + "peers_onion": packed_peers["onion"] + } + + self.response(back) + + # Get modified content.json files since + def actionListModified(self, params): + site = self.sites.get(params["site"]) + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(5) + return False + modified_files = site.content_manager.listModified(params["since"]) + + # Add peer to site if not added before + connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") + if connected_peer: # Just added + connected_peer.connect(self.connection) # Assign current connection to peer + + self.response({"modified_files": modified_files}) + + def actionGetHashfield(self, params): + site = self.sites.get(params["site"]) + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(5) + return False + + # Add peer to site if not added before + peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="request") + if not peer.connection: # Just added + peer.connect(self.connection) # Assign current connection to peer + + peer.time_my_hashfield_sent = time.time() # Don't send again if not changed + + self.response({"hashfield_raw": site.content_manager.hashfield.tobytes()}) + + def findHashIds(self, site, hash_ids, limit=100): + back = collections.defaultdict(lambda: collections.defaultdict(list)) + found = site.worker_manager.findOptionalHashIds(hash_ids, limit=limit) + + for hash_id, peers in found.items(): + for peer in peers: + ip_type = helper.getIpType(peer.ip) + if len(back[ip_type][hash_id]) < 20: + back[ip_type][hash_id].append(peer.packMyAddress()) + return back + + def actionFindHashIds(self, params): + site = self.sites.get(params["site"]) + s = time.time() + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(5) + return False + + event_key = "%s_findHashIds_%s_%s" % (self.connection.ip, params["site"], len(params["hash_ids"])) + if self.connection.cpu_time > 0.5 or not RateLimit.isAllowed(event_key, 60 * 5): + time.sleep(0.1) + back = self.findHashIds(site, params["hash_ids"], limit=10) + else: + back = self.findHashIds(site, params["hash_ids"]) + RateLimit.called(event_key) + + my_hashes = [] + my_hashfield_set = set(site.content_manager.hashfield) + for hash_id in params["hash_ids"]: + if hash_id in my_hashfield_set: + my_hashes.append(hash_id) + + if config.verbose: + self.log.debug( + "Found: %s for %s hashids in %.3fs" % + ({key: len(val) for key, val in back.items()}, len(params["hash_ids"]), time.time() - s) + ) + self.response({"peers": back["ipv4"], "peers_onion": back["onion"], "peers_ipv6": back["ipv6"], "my": my_hashes}) + + def actionSetHashfield(self, params): + site = self.sites.get(params["site"]) + if not site or not site.isServing(): # Site unknown or not serving + self.response({"error": "Unknown site"}) + self.connection.badAction(5) + return False + + # Add or get peer + peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, connection=self.connection, source="request") + if not peer.connection: + peer.connect(self.connection) + peer.hashfield.replaceFromBytes(params["hashfield_raw"]) + self.response({"ok": "Updated"}) + + # Send a simple Pong! answer + def actionPing(self, params): + self.response(b"Pong!") + + # Check requested port of the other peer + def actionCheckport(self, params): + if helper.getIpType(self.connection.ip) == "ipv6": + sock_address = (self.connection.ip, params["port"], 0, 0) + else: + sock_address = (self.connection.ip, params["port"]) + + with closing(helper.createSocket(self.connection.ip)) as sock: + sock.settimeout(5) + if sock.connect_ex(sock_address) == 0: + self.response({"status": "open", "ip_external": self.connection.ip}) + else: + self.response({"status": "closed", "ip_external": self.connection.ip}) + + # Unknown command + def actionUnknown(self, cmd, params): + self.response({"error": "Unknown command: %s" % cmd}) + self.connection.badAction(5) diff --git a/src/File/FileServer.py b/src/File/FileServer.py new file mode 100644 index 00000000..b7a942fc --- /dev/null +++ b/src/File/FileServer.py @@ -0,0 +1,409 @@ +import logging +import time +import random +import socket +import sys + +import gevent +import gevent.pool +from gevent.server import StreamServer + +import util +from util import helper +from Config import config +from .FileRequest import FileRequest +from Peer import PeerPortchecker +from Site import SiteManager +from Connection import ConnectionServer +from Plugin import PluginManager +from Debug import Debug + + +@PluginManager.acceptPlugins +class FileServer(ConnectionServer): + + def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port, ip_type=config.fileserver_ip_type): + self.site_manager = SiteManager.site_manager + self.portchecker = PeerPortchecker.PeerPortchecker(self) + self.log = logging.getLogger("FileServer") + self.ip_type = ip_type + self.ip_external_list = [] + + self.supported_ip_types = ["ipv4"] # Outgoing ip_type support + if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): + self.supported_ip_types.append("ipv6") + + if ip_type == "ipv6" or (ip_type == "dual" and "ipv6" in self.supported_ip_types): + ip = ip.replace("*", "::") + else: + ip = ip.replace("*", "0.0.0.0") + + if config.tor == "always": + port = config.tor_hs_port + config.fileserver_port = port + elif port == 0: # Use random port + port_range_from, port_range_to = list(map(int, config.fileserver_port_range.split("-"))) + port = self.getRandomPort(ip, port_range_from, port_range_to) + config.fileserver_port = port + if not port: + raise Exception("Can't find bindable port") + if not config.tor == "always": + config.saveValue("fileserver_port", port) # Save random port value for next restart + config.arguments.fileserver_port = port + + ConnectionServer.__init__(self, ip, port, self.handleRequest) + self.log.debug("Supported IP types: %s" % self.supported_ip_types) + + if ip_type == "dual" and ip == "::": + # Also bind to ipv4 addres in dual mode + try: + self.log.debug("Binding proxy to %s:%s" % ("::", self.port)) + self.stream_server_proxy = StreamServer( + ("0.0.0.0", self.port), self.handleIncomingConnection, spawn=self.pool, backlog=100 + ) + except Exception as err: + self.log.info("StreamServer proxy create error: %s" % Debug.formatException(err)) + + self.port_opened = {} + + self.sites = self.site_manager.sites + self.last_request = time.time() + self.files_parsing = {} + self.ui_server = None + + def getRandomPort(self, ip, port_range_from, port_range_to): + """Generates Random Port from given range + Args: + ip: IP Address + port_range_from: From Range + port_range_to: to Range + """ + self.log.info("Getting random port in range %s-%s..." % (port_range_from, port_range_to)) + tried = [] + for bind_retry in range(100): + port = random.randint(port_range_from, port_range_to) + if port in tried: + continue + tried.append(port) + sock = helper.createSocket(ip) + try: + sock.bind((ip, port)) + success = True + except Exception as err: + self.log.warning("Error binding to port %s: %s" % (port, err)) + success = False + sock.close() + if success: + self.log.info("Found unused random port: %s" % port) + return port + else: + time.sleep(0.1) + return False + + def isIpv6Supported(self): + if config.tor == "always": + return True + # Test if we can connect to ipv6 address + ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" + try: + sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + sock.connect((ipv6_testip, 80)) + local_ipv6 = sock.getsockname()[0] + if local_ipv6 == "::1": + self.log.debug("IPv6 not supported, no local IPv6 address") + return False + else: + self.log.debug("IPv6 supported on IP %s" % local_ipv6) + return True + except socket.error as err: + self.log.warning("IPv6 not supported: %s" % err) + return False + except Exception as err: + self.log.error("IPv6 check error: %s" % err) + return False + + def listenProxy(self): + try: + self.stream_server_proxy.serve_forever() + except Exception as err: + if err.errno == 98: # Address already in use error + self.log.debug("StreamServer proxy listen error: %s" % err) + else: + self.log.info("StreamServer proxy listen error: %s" % err) + + # Handle request to fileserver + def handleRequest(self, connection, message): + if config.verbose: + if "params" in message: + self.log.debug( + "FileRequest: %s %s %s %s" % + (str(connection), message["cmd"], message["params"].get("site"), message["params"].get("inner_path")) + ) + else: + self.log.debug("FileRequest: %s %s" % (str(connection), message["cmd"])) + req = FileRequest(self, connection) + req.route(message["cmd"], message.get("req_id"), message.get("params")) + if not self.has_internet and not connection.is_private_ip: + self.has_internet = True + self.onInternetOnline() + + def onInternetOnline(self): + self.log.info("Internet online") + gevent.spawn(self.checkSites, check_files=False, force_port_check=True) + + # Reload the FileRequest class to prevent restarts in debug mode + def reload(self): + global FileRequest + import imp + FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest + + def portCheck(self): + if config.offline: + self.log.info("Offline mode: port check disabled") + res = {"ipv4": None, "ipv6": None} + self.port_opened = res + return res + + if config.ip_external: + for ip_external in config.ip_external: + SiteManager.peer_blacklist.append((ip_external, self.port)) # Add myself to peer blacklist + + ip_external_types = set([helper.getIpType(ip) for ip in config.ip_external]) + res = { + "ipv4": "ipv4" in ip_external_types, + "ipv6": "ipv6" in ip_external_types + } + self.ip_external_list = config.ip_external + self.port_opened.update(res) + self.log.info("Server port opened based on configuration ipv4: %s, ipv6: %s" % (res["ipv4"], res["ipv6"])) + return res + + self.port_opened = {} + if self.ui_server: + self.ui_server.updateWebsocket() + + if "ipv6" in self.supported_ip_types: + res_ipv6_thread = gevent.spawn(self.portchecker.portCheck, self.port, "ipv6") + else: + res_ipv6_thread = None + + res_ipv4 = self.portchecker.portCheck(self.port, "ipv4") + if not res_ipv4["opened"] and config.tor != "always": + if self.portchecker.portOpen(self.port): + res_ipv4 = self.portchecker.portCheck(self.port, "ipv4") + + if res_ipv6_thread is None: + res_ipv6 = {"ip": None, "opened": None} + else: + res_ipv6 = res_ipv6_thread.get() + if res_ipv6["opened"] and not helper.getIpType(res_ipv6["ip"]) == "ipv6": + self.log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"]) + res_ipv6["opened"] = False + + self.ip_external_list = [] + for res_ip in [res_ipv4, res_ipv6]: + if res_ip["ip"] and res_ip["ip"] not in self.ip_external_list: + self.ip_external_list.append(res_ip["ip"]) + SiteManager.peer_blacklist.append((res_ip["ip"], self.port)) + + self.log.info("Server port opened ipv4: %s, ipv6: %s" % (res_ipv4["opened"], res_ipv6["opened"])) + + res = {"ipv4": res_ipv4["opened"], "ipv6": res_ipv6["opened"]} + + # Add external IPs from local interfaces + interface_ips = helper.getInterfaceIps("ipv4") + if "ipv6" in self.supported_ip_types: + interface_ips += helper.getInterfaceIps("ipv6") + for ip in interface_ips: + if not helper.isPrivateIp(ip) and ip not in self.ip_external_list: + self.ip_external_list.append(ip) + res[helper.getIpType(ip)] = True # We have opened port if we have external ip + SiteManager.peer_blacklist.append((ip, self.port)) + self.log.debug("External ip found on interfaces: %s" % ip) + + self.port_opened.update(res) + + if self.ui_server: + self.ui_server.updateWebsocket() + + return res + + # Check site file integrity + def checkSite(self, site, check_files=False): + if site.isServing(): + site.announce(mode="startup") # Announce site to tracker + site.update(check_files=check_files) # Update site's content.json and download changed files + site.sendMyHashfield() + site.updateHashfield() + + # Check sites integrity + @util.Noparallel() + def checkSites(self, check_files=False, force_port_check=False): + self.log.debug("Checking sites...") + s = time.time() + sites_checking = False + if not self.port_opened or force_port_check: # Test and open port if not tested yet + if len(self.sites) <= 2: # Don't wait port opening on first startup + sites_checking = True + for address, site in list(self.sites.items()): + gevent.spawn(self.checkSite, site, check_files) + + self.portCheck() + + if not self.port_opened["ipv4"]: + self.tor_manager.startOnions() + + if not sites_checking: + check_pool = gevent.pool.Pool(5) + # Check sites integrity + for site in sorted(list(self.sites.values()), key=lambda site: site.settings.get("modified", 0), reverse=True): + if not site.isServing(): + continue + check_thread = check_pool.spawn(self.checkSite, site, check_files) # Check in new thread + time.sleep(2) + if site.settings.get("modified", 0) < time.time() - 60 * 60 * 24: # Not so active site, wait some sec to finish + check_thread.join(timeout=5) + self.log.debug("Checksites done in %.3fs" % (time.time() - s)) + + def cleanupSites(self): + import gc + startup = True + time.sleep(5 * 60) # Sites already cleaned up on startup + peers_protected = set([]) + while 1: + # Sites health care every 20 min + self.log.debug( + "Running site cleanup, connections: %s, internet: %s, protected peers: %s" % + (len(self.connections), self.has_internet, len(peers_protected)) + ) + + for address, site in list(self.sites.items()): + if not site.isServing(): + continue + + if not startup: + site.cleanupPeers(peers_protected) + + time.sleep(1) # Prevent too quick request + + peers_protected = set([]) + for address, site in list(self.sites.items()): + if not site.isServing(): + continue + + if site.peers: + with gevent.Timeout(10, exception=False): + site.announcer.announcePex() + + # Last check modification failed + if site.content_updated is False: + site.update() + elif site.bad_files: + site.retryBadFiles() + + if time.time() - site.settings.get("modified", 0) < 60 * 60 * 24 * 7: + # Keep active connections if site has been modified witin 7 days + connected_num = site.needConnections(check_site_on_reconnect=True) + + if connected_num < config.connected_limit: # This site has small amount of peers, protect them from closing + peers_protected.update([peer.key for peer in site.getConnectedPeers()]) + + time.sleep(1) # Prevent too quick request + + site = None + gc.collect() # Implicit garbage collection + startup = False + time.sleep(60 * 20) + + def announceSite(self, site): + site.announce(mode="update", pex=False) + active_site = time.time() - site.settings.get("modified", 0) < 24 * 60 * 60 + if site.settings["own"] or active_site: + # Check connections more frequently on own and active sites to speed-up first connections + site.needConnections(check_site_on_reconnect=True) + site.sendMyHashfield(3) + site.updateHashfield(3) + + # Announce sites every 20 min + def announceSites(self): + time.sleep(5 * 60) # Sites already announced on startup + while 1: + config.loadTrackersFile() + s = time.time() + for address, site in list(self.sites.items()): + if not site.isServing(): + continue + gevent.spawn(self.announceSite, site).join(timeout=10) + time.sleep(1) + taken = time.time() - s + + # Query all trackers one-by-one in 20 minutes evenly distributed + sleep = max(0, 60 * 20 / len(config.trackers) - taken) + + self.log.debug("Site announce tracker done in %.3fs, sleeping for %.3fs..." % (taken, sleep)) + time.sleep(sleep) + + # Detects if computer back from wakeup + def wakeupWatcher(self): + last_time = time.time() + last_my_ips = socket.gethostbyname_ex('')[2] + while 1: + time.sleep(30) + is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3 + if is_time_changed: + # If taken more than 3 minute then the computer was in sleep mode + self.log.info( + "Wakeup detected: time warp from %0.f to %0.f (%0.f sleep seconds), acting like startup..." % + (last_time, time.time(), time.time() - last_time) + ) + + my_ips = socket.gethostbyname_ex('')[2] + is_ip_changed = my_ips != last_my_ips + if is_ip_changed: + self.log.info("IP change detected from %s to %s" % (last_my_ips, my_ips)) + + if is_time_changed or is_ip_changed: + self.checkSites(check_files=False, force_port_check=True) + + last_time = time.time() + last_my_ips = my_ips + + # Bind and start serving sites + def start(self, check_sites=True): + if self.stopping: + return False + + ConnectionServer.start(self) + + try: + self.stream_server.start() + except Exception as err: + self.log.error("Error listening on: %s:%s: %s" % (self.ip, self.port, err)) + + self.sites = self.site_manager.list() + if config.debug: + # Auto reload FileRequest on change + from Debug import DebugReloader + DebugReloader.watcher.addCallback(self.reload) + + if check_sites: # Open port, Update sites, Check files integrity + gevent.spawn(self.checkSites) + + thread_announce_sites = gevent.spawn(self.announceSites) + thread_cleanup_sites = gevent.spawn(self.cleanupSites) + thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) + + ConnectionServer.listen(self) + + self.log.debug("Stopped.") + + def stop(self): + if self.running and self.portchecker.upnp_port_opened: + self.log.debug('Closing port %d' % self.port) + try: + self.portchecker.portClose(self.port) + self.log.info('Closed port via upnp.') + except Exception as err: + self.log.info("Failed at attempt to use upnp to close port: %s" % err) + + return ConnectionServer.stop(self) diff --git a/src/File/__init__.py b/src/File/__init__.py new file mode 100644 index 00000000..1eb602d6 --- /dev/null +++ b/src/File/__init__.py @@ -0,0 +1,2 @@ +from .FileServer import FileServer +from .FileRequest import FileRequest \ No newline at end of file diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py new file mode 100644 index 00000000..03cc1f47 --- /dev/null +++ b/src/Peer/Peer.py @@ -0,0 +1,410 @@ +import logging +import time +import sys +import itertools +import collections + +import gevent + +import io +from Debug import Debug +from Config import config +from util import helper +from .PeerHashfield import PeerHashfield +from Plugin import PluginManager + +if config.use_tempfiles: + import tempfile + + +# Communicate remote peers +@PluginManager.acceptPlugins +class Peer(object): + __slots__ = ( + "ip", "port", "site", "key", "connection", "connection_server", "time_found", "time_response", "time_hashfield", + "time_added", "has_hashfield", "is_tracker_connection", "time_my_hashfield_sent", "last_ping", "reputation", + "last_content_json_update", "hashfield", "connection_error", "hash_failed", "download_bytes", "download_time" + ) + + def __init__(self, ip, port, site=None, connection_server=None): + self.ip = ip + self.port = port + self.site = site + self.key = "%s:%s" % (ip, port) + + self.connection = None + self.connection_server = connection_server + self.has_hashfield = False # Lazy hashfield object not created yet + self.time_hashfield = None # Last time peer's hashfiled downloaded + self.time_my_hashfield_sent = None # Last time my hashfield sent to peer + self.time_found = time.time() # Time of last found in the torrent tracker + self.time_response = None # Time of last successful response from peer + self.time_added = time.time() + self.last_ping = None # Last response time for ping + self.is_tracker_connection = False # Tracker connection instead of normal peer + self.reputation = 0 # More likely to connect if larger + self.last_content_json_update = 0.0 # Modify date of last received content.json + + self.connection_error = 0 # Series of connection error + self.hash_failed = 0 # Number of bad files from peer + self.download_bytes = 0 # Bytes downloaded + self.download_time = 0 # Time spent to download + + def __getattr__(self, key): + if key == "hashfield": + self.has_hashfield = True + self.hashfield = PeerHashfield() + return self.hashfield + else: + return getattr(self, key) + + def log(self, text): + if not config.verbose: + return # Only log if we are in debug mode + if self.site: + self.site.log.debug("%s:%s %s" % (self.ip, self.port, text)) + else: + logging.debug("%s:%s %s" % (self.ip, self.port, text)) + + # Connect to host + def connect(self, connection=None): + if self.reputation < -10: + self.reputation = -10 + if self.reputation > 10: + self.reputation = 10 + + if self.connection: + self.log("Getting connection (Closing %s)..." % self.connection) + self.connection.close("Connection change") + else: + self.log("Getting connection (reputation: %s)..." % self.reputation) + + if connection: # Connection specified + self.log("Assigning connection %s" % connection) + self.connection = connection + self.connection.sites += 1 + else: # Try to find from connection pool or create new connection + self.connection = None + + try: + if self.connection_server: + connection_server = self.connection_server + elif self.site: + connection_server = self.site.connection_server + else: + import main + connection_server = main.file_server + self.connection = connection_server.getConnection(self.ip, self.port, site=self.site, is_tracker_connection=self.is_tracker_connection) + self.reputation += 1 + self.connection.sites += 1 + except Exception as err: + self.onConnectionError("Getting connection error") + self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % + (Debug.formatException(err), self.connection_error, self.hash_failed)) + self.connection = None + return self.connection + + # Check if we have connection to peer + def findConnection(self): + if self.connection and self.connection.connected: # We have connection to peer + return self.connection + else: # Try to find from other sites connections + self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False, site=self.site) + if self.connection: + self.connection.sites += 1 + return self.connection + + def __str__(self): + if self.site: + return "Peer:%-12s of %s" % (self.ip, self.site.address_short) + else: + return "Peer:%-12s" % self.ip + + def __repr__(self): + return "<%s>" % self.__str__() + + def packMyAddress(self): + if self.ip.endswith(".onion"): + return helper.packOnionAddress(self.ip, self.port) + else: + return helper.packAddress(self.ip, self.port) + + # Found a peer from a source + def found(self, source="other"): + if self.reputation < 5: + if source == "tracker": + if self.ip.endswith(".onion"): + self.reputation += 1 + else: + self.reputation += 2 + elif source == "local": + self.reputation += 20 + + if source in ("tracker", "local"): + self.site.peers_recent.appendleft(self) + self.time_found = time.time() + + # Send a command to peer and return response value + def request(self, cmd, params={}, stream_to=None): + if not self.connection or self.connection.closed: + self.connect() + if not self.connection: + self.onConnectionError("Reconnect error") + return None # Connection failed + + self.log("Send request: %s %s %s %s" % (params.get("site", ""), cmd, params.get("inner_path", ""), params.get("location", ""))) + + for retry in range(1, 4): # Retry 3 times + try: + if not self.connection: + raise Exception("No connection found") + res = self.connection.request(cmd, params, stream_to) + if not res: + raise Exception("Send error") + if "error" in res: + self.log("%s error: %s" % (cmd, res["error"])) + self.onConnectionError("Response error") + break + else: # Successful request, reset connection error num + self.connection_error = 0 + self.time_response = time.time() + if res: + return res + else: + raise Exception("Invalid response: %s" % res) + except Exception as err: + if type(err).__name__ == "Notify": # Greenlet killed by worker + self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd)) + break + else: + self.onConnectionError("Request error") + self.log( + "%s (connection_error: %s, hash_failed: %s, retry: %s)" % + (Debug.formatException(err), self.connection_error, self.hash_failed, retry) + ) + time.sleep(1 * retry) + self.connect() + return None # Failed after 4 retry + + # Get a file content from peer + def getFile(self, site, inner_path, file_size=None, pos_from=0, pos_to=None, streaming=False): + if file_size and file_size > 5 * 1024 * 1024: + max_read_size = 1024 * 1024 + else: + max_read_size = 512 * 1024 + + if pos_to: + read_bytes = min(max_read_size, pos_to - pos_from) + else: + read_bytes = max_read_size + + location = pos_from + + if config.use_tempfiles: + buff = tempfile.SpooledTemporaryFile(max_size=16 * 1024, mode='w+b') + else: + buff = io.BytesIO() + + s = time.time() + while True: # Read in smaller parts + if config.stream_downloads or read_bytes > 256 * 1024 or streaming: + res = self.request("streamFile", {"site": site, "inner_path": inner_path, "location": location, "read_bytes": read_bytes, "file_size": file_size}, stream_to=buff) + if not res or "location" not in res: # Error + return False + else: + self.log("Send: %s" % inner_path) + res = self.request("getFile", {"site": site, "inner_path": inner_path, "location": location, "read_bytes": read_bytes, "file_size": file_size}) + if not res or "location" not in res: # Error + return False + self.log("Recv: %s" % inner_path) + buff.write(res["body"]) + res["body"] = None # Save memory + + if res["location"] == res["size"] or res["location"] == pos_to: # End of file + break + else: + location = res["location"] + if pos_to: + read_bytes = min(max_read_size, pos_to - location) + + if pos_to: + recv = pos_to - pos_from + else: + recv = res["location"] + + self.download_bytes += recv + self.download_time += (time.time() - s) + if self.site: + self.site.settings["bytes_recv"] = self.site.settings.get("bytes_recv", 0) + recv + self.log("Downloaded: %s, pos: %s, read_bytes: %s" % (inner_path, buff.tell(), read_bytes)) + buff.seek(0) + return buff + + # Send a ping request + def ping(self): + response_time = None + for retry in range(1, 3): # Retry 3 times + s = time.time() + with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception + res = self.request("ping") + + if res and "body" in res and res["body"] == b"Pong!": + response_time = time.time() - s + break # All fine, exit from for loop + # Timeout reached or bad response + self.onConnectionError("Ping timeout") + self.connect() + time.sleep(1) + + if response_time: + self.log("Ping: %.3f" % response_time) + else: + self.log("Ping failed") + self.last_ping = response_time + return response_time + + # Request peer exchange from peer + def pex(self, site=None, need_num=5): + if not site: + site = self.site # If no site defined request peers for this site + + # give back 5 connectible peers + packed_peers = helper.packPeers(self.site.getConnectablePeers(5, allow_private=False)) + request = {"site": site.address, "peers": packed_peers["ipv4"], "need": need_num} + if packed_peers["onion"]: + request["peers_onion"] = packed_peers["onion"] + if packed_peers["ipv6"]: + request["peers_ipv6"] = packed_peers["ipv6"] + res = self.request("pex", request) + if not res or "error" in res: + return False + added = 0 + + # Remove unsupported peer types + if "peers_ipv6" in res and self.connection and "ipv6" not in self.connection.server.supported_ip_types: + del res["peers_ipv6"] + + if "peers_onion" in res and self.connection and "onion" not in self.connection.server.supported_ip_types: + del res["peers_onion"] + + # Add IPv4 + IPv6 + for peer in itertools.chain(res.get("peers", []), res.get("peers_ipv6", [])): + address = helper.unpackAddress(peer) + if site.addPeer(*address, source="pex"): + added += 1 + + # Add Onion + for peer in res.get("peers_onion", []): + address = helper.unpackOnionAddress(peer) + if site.addPeer(*address, source="pex"): + added += 1 + + if added: + self.log("Added peers using pex: %s" % added) + + return added + + # List modified files since the date + # Return: {inner_path: modification date,...} + def listModified(self, since): + return self.request("listModified", {"since": since, "site": self.site.address}) + + def updateHashfield(self, force=False): + # Don't update hashfield again in 5 min + if self.time_hashfield and time.time() - self.time_hashfield < 5 * 60 and not force: + return False + + self.time_hashfield = time.time() + res = self.request("getHashfield", {"site": self.site.address}) + if not res or "error" in res or "hashfield_raw" not in res: + return False + self.hashfield.replaceFromBytes(res["hashfield_raw"]) + + return self.hashfield + + # Find peers for hashids + # Return: {hash1: ["ip:port", "ip:port",...],...} + def findHashIds(self, hash_ids): + res = self.request("findHashIds", {"site": self.site.address, "hash_ids": hash_ids}) + if not res or "error" in res or type(res) is not dict: + return False + + back = collections.defaultdict(list) + + for ip_type in ["ipv4", "ipv6", "onion"]: + if ip_type == "ipv4": + key = "peers" + else: + key = "peers_%s" % ip_type + for hash, peers in list(res.get(key, {}).items())[0:30]: + if ip_type == "onion": + unpacker_func = helper.unpackOnionAddress + else: + unpacker_func = helper.unpackAddress + + back[hash] += list(map(unpacker_func, peers)) + + for hash in res.get("my", []): + if self.connection: + back[hash].append((self.connection.ip, self.connection.port)) + else: + back[hash].append((self.ip, self.port)) + + return back + + # Send my hashfield to peer + # Return: True if sent + def sendMyHashfield(self): + if self.connection and self.connection.handshake.get("rev", 0) < 510: + return False # Not supported + if self.time_my_hashfield_sent and self.site.content_manager.hashfield.time_changed <= self.time_my_hashfield_sent: + return False # Peer already has the latest hashfield + + res = self.request("setHashfield", {"site": self.site.address, "hashfield_raw": self.site.content_manager.hashfield.tobytes()}) + if not res or "error" in res: + return False + else: + self.time_my_hashfield_sent = time.time() + return True + + def publish(self, address, inner_path, body, modified, diffs=[]): + if len(body) > 10 * 1024 and self.connection and self.connection.handshake.get("rev", 0) >= 4095: + # To save bw we don't push big content.json to peers + body = b"" + + return self.request("update", { + "site": address, + "inner_path": inner_path, + "body": body, + "modified": modified, + "diffs": diffs + }) + + # Stop and remove from site + def remove(self, reason="Removing"): + self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed)) + if self.site and self.key in self.site.peers: + del(self.site.peers[self.key]) + + if self.site and self in self.site.peers_recent: + self.site.peers_recent.remove(self) + + if self.connection: + self.connection.close(reason) + + # - EVENTS - + + # On connection error + def onConnectionError(self, reason="Unknown"): + self.connection_error += 1 + if self.site and len(self.site.peers) > 200: + limit = 3 + else: + limit = 6 + self.reputation -= 1 + if self.connection_error >= limit: # Dead peer + self.remove("Peer connection: %s" % reason) + + # Done working with peer + def onWorkerDone(self): + pass diff --git a/src/Peer/PeerHashfield.py b/src/Peer/PeerHashfield.py new file mode 100644 index 00000000..fdd414c8 --- /dev/null +++ b/src/Peer/PeerHashfield.py @@ -0,0 +1,75 @@ +import array +import time + + +class PeerHashfield(object): + __slots__ = ("storage", "time_changed", "append", "remove", "tobytes", "frombytes", "__len__", "__iter__") + def __init__(self): + self.storage = self.createStorage() + self.time_changed = time.time() + + def createStorage(self): + storage = array.array("H") + self.append = storage.append + self.remove = storage.remove + self.tobytes = storage.tobytes + self.frombytes = storage.frombytes + self.__len__ = storage.__len__ + self.__iter__ = storage.__iter__ + return storage + + def appendHash(self, hash): + hash_id = int(hash[0:4], 16) + if hash_id not in self.storage: + self.storage.append(hash_id) + self.time_changed = time.time() + return True + else: + return False + + def appendHashId(self, hash_id): + if hash_id not in self.storage: + self.storage.append(hash_id) + self.time_changed = time.time() + return True + else: + return False + + def removeHash(self, hash): + hash_id = int(hash[0:4], 16) + if hash_id in self.storage: + self.storage.remove(hash_id) + self.time_changed = time.time() + return True + else: + return False + + def removeHashId(self, hash_id): + if hash_id in self.storage: + self.storage.remove(hash_id) + self.time_changed = time.time() + return True + else: + return False + + def getHashId(self, hash): + return int(hash[0:4], 16) + + def hasHash(self, hash): + return int(hash[0:4], 16) in self.storage + + def replaceFromBytes(self, hashfield_raw): + self.storage = self.createStorage() + self.storage.frombytes(hashfield_raw) + self.time_changed = time.time() + +if __name__ == "__main__": + field = PeerHashfield() + s = time.time() + for i in range(10000): + field.appendHashId(i) + print(time.time()-s) + s = time.time() + for i in range(10000): + field.hasHash("AABB") + print(time.time()-s) \ No newline at end of file diff --git a/src/Peer/PeerPortchecker.py b/src/Peer/PeerPortchecker.py new file mode 100644 index 00000000..3c4daecf --- /dev/null +++ b/src/Peer/PeerPortchecker.py @@ -0,0 +1,189 @@ +import logging +import urllib.request +import urllib.parse +import re +import time + +from Debug import Debug +from util import UpnpPunch + + +class PeerPortchecker(object): + checker_functions = { + "ipv4": ["checkIpfingerprints", "checkCanyouseeme"], + "ipv6": ["checkMyaddr", "checkIpv6scanner"] + } + def __init__(self, file_server): + self.log = logging.getLogger("PeerPortchecker") + self.upnp_port_opened = False + self.file_server = file_server + + def requestUrl(self, url, post_data=None): + if type(post_data) is dict: + post_data = urllib.parse.urlencode(post_data).encode("utf8") + req = urllib.request.Request(url, post_data) + req.add_header("Referer", url) + req.add_header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") + req.add_header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + return urllib.request.urlopen(req, timeout=20.0) + + def portOpen(self, port): + self.log.info("Trying to open port using UpnpPunch...") + + try: + UpnpPunch.ask_to_open_port(port, 'ZeroNet', retries=3, protos=["TCP"]) + self.upnp_port_opened = True + except Exception as err: + self.log.warning("UpnpPunch run error: %s" % Debug.formatException(err)) + return False + + return True + + def portClose(self, port): + return UpnpPunch.ask_to_close_port(port, protos=["TCP"]) + + def portCheck(self, port, ip_type="ipv4"): + checker_functions = self.checker_functions[ip_type] + + for func_name in checker_functions: + func = getattr(self, func_name) + s = time.time() + try: + res = func(port) + if res: + self.log.info( + "Checked port %s (%s) using %s result: %s in %.3fs" % + (port, ip_type, func_name, res, time.time() - s) + ) + time.sleep(0.1) + if res["opened"] and not self.file_server.had_external_incoming: + res["opened"] = False + self.log.warning("Port %s:%s looks opened, but no incoming connection" % (res["ip"], port)) + break + except Exception as err: + self.log.warning( + "%s check error: %s in %.3fs" % + (func_name, Debug.formatException(err), time.time() - s) + ) + res = {"ip": None, "opened": False} + + return res + + def checkCanyouseeme(self, port): + data = urllib.request.urlopen("https://www.canyouseeme.org/", b"ip=1.1.1.1&port=%s" % str(port).encode("ascii"), timeout=20.0).read().decode("utf8") + + message = re.match(r'.*

(.*?)

', data, re.DOTALL).group(1) + message = re.sub(r"<.*?>", "", message.replace("
", " ").replace(" ", " ")) # Strip http tags + + match = re.match(r".*service on (.*?) on", message) + if match: + ip = match.group(1) + else: + raise Exception("Invalid response: %s" % message) + + if "Success" in message: + return {"ip": ip, "opened": True} + elif "Error" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkIpfingerprints(self, port): + data = self.requestUrl("https://www.ipfingerprints.com/portscan.php").read().decode("utf8") + ip = re.match(r'.*name="remoteHost".*?value="(.*?)"', data, re.DOTALL).group(1) + + post_data = { + "remoteHost": ip, "start_port": port, "end_port": port, + "normalScan": "Yes", "scan_type": "connect2", "ping_type": "none" + } + message = self.requestUrl("https://www.ipfingerprints.com/scripts/getPortsInfo.php", post_data).read().decode("utf8") + + if "open" in message: + return {"ip": ip, "opened": True} + elif "filtered" in message or "closed" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkMyaddr(self, port): + url = "http://ipv6.my-addr.com/online-ipv6-port-scan.php" + + data = self.requestUrl(url).read().decode("utf8") + + ip = re.match(r'.*Your IP address is:[ ]*([0-9\.:a-z]+)', data.replace(" ", ""), re.DOTALL).group(1) + + post_data = {"addr": ip, "ports_selected": "", "ports_list": port} + data = self.requestUrl(url, post_data).read().decode("utf8") + + message = re.match(r".*(.*?)
", data, re.DOTALL).group(1) + + if "ok.png" in message: + return {"ip": ip, "opened": True} + elif "fail.png" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkIpv6scanner(self, port): + url = "http://www.ipv6scanner.com/cgi-bin/main.py" + + data = self.requestUrl(url).read().decode("utf8") + + ip = re.match(r'.*Your IP address is[ ]*([0-9\.:a-z]+)', data.replace(" ", ""), re.DOTALL).group(1) + + post_data = {"host": ip, "scanType": "1", "port": port, "protocol": "tcp", "authorized": "yes"} + data = self.requestUrl(url, post_data).read().decode("utf8") + + message = re.match(r".*(.*?)
", data, re.DOTALL).group(1) + message_text = re.sub("<.*?>", " ", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags + + if "OPEN" in message_text: + return {"ip": ip, "opened": True} + elif "CLOSED" in message_text or "FILTERED" in message_text: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message_text) + + def checkPortchecker(self, port): # Not working: Forbidden + data = self.requestUrl("https://portchecker.co").read().decode("utf8") + csrf = re.match(r'.*name="_csrf" value="(.*?)"', data, re.DOTALL).group(1) + + data = self.requestUrl("https://portchecker.co", {"port": port, "_csrf": csrf}).read().decode("utf8") + message = re.match(r'.*
(.*?)
', data, re.DOTALL).group(1) + message = re.sub(r"<.*?>", "", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags + + match = re.match(r".*targetIP.*?value=\"(.*?)\"", data, re.DOTALL) + if match: + ip = match.group(1) + else: + raise Exception("Invalid response: %s" % message) + + if "open" in message: + return {"ip": ip, "opened": True} + elif "closed" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) + + def checkSubnetonline(self, port): # Not working: Invalid response + url = "https://www.subnetonline.com/pages/ipv6-network-tools/online-ipv6-port-scanner.php" + + data = self.requestUrl(url).read().decode("utf8") + + ip = re.match(r'.*Your IP is.*?name="host".*?value="(.*?)"', data, re.DOTALL).group(1) + token = re.match(r'.*name="token".*?value="(.*?)"', data, re.DOTALL).group(1) + + post_data = {"host": ip, "port": port, "allow": "on", "token": token, "submit": "Scanning.."} + data = self.requestUrl(url, post_data).read().decode("utf8") + + print(post_data, data) + + message = re.match(r".*
(.*?)
", data, re.DOTALL).group(1) + message = re.sub(r"<.*?>", "", message.replace("
", " ").replace(" ", " ").strip()) # Strip http tags + + if "online" in message: + return {"ip": ip, "opened": True} + elif "closed" in message: + return {"ip": ip, "opened": False} + else: + raise Exception("Invalid response: %s" % message) diff --git a/src/Peer/__init__.py b/src/Peer/__init__.py new file mode 100644 index 00000000..e73c58c5 --- /dev/null +++ b/src/Peer/__init__.py @@ -0,0 +1,2 @@ +from .Peer import Peer +from .PeerHashfield import PeerHashfield diff --git a/src/Plugin/PluginManager.py b/src/Plugin/PluginManager.py new file mode 100644 index 00000000..56540e60 --- /dev/null +++ b/src/Plugin/PluginManager.py @@ -0,0 +1,292 @@ +import logging +import os +import sys +import shutil +import time +from collections import defaultdict + +import importlib +import json + +from Debug import Debug +from Config import config +import plugins + + +class PluginManager: + def __init__(self): + self.log = logging.getLogger("PluginManager") + self.path_plugins = None + if plugins.__file__: + self.path_plugins = os.path.dirname(os.path.abspath(plugins.__file__)); + self.path_installed_plugins = config.data_dir + "/__plugins__" + self.plugins = defaultdict(list) # Registered plugins (key: class name, value: list of plugins for class) + self.subclass_order = {} # Record the load order of the plugins, to keep it after reload + self.pluggable = {} + self.plugin_names = [] # Loaded plugin names + self.plugins_updated = {} # List of updated plugins since restart + self.plugins_rev = {} # Installed plugins revision numbers + self.after_load = [] # Execute functions after loaded plugins + self.function_flags = {} # Flag function for permissions + self.reloading = False + self.config_path = config.data_dir + "/plugins.json" + self.loadConfig() + + self.config.setdefault("builtin", {}) + + if self.path_plugins: + sys.path.append(os.path.join(os.getcwd(), self.path_plugins)) + self.migratePlugins() + + if config.debug: # Auto reload Plugins on file change + from Debug import DebugReloader + DebugReloader.watcher.addCallback(self.reloadPlugins) + + def loadConfig(self): + if os.path.isfile(self.config_path): + try: + self.config = json.load(open(self.config_path, encoding="utf8")) + except Exception as err: + self.log.error("Error loading %s: %s" % (self.config_path, err)) + self.config = {} + else: + self.config = {} + + def saveConfig(self): + f = open(self.config_path, "w", encoding="utf8") + json.dump(self.config, f, ensure_ascii=False, sort_keys=True, indent=2) + + def migratePlugins(self): + for dir_name in os.listdir(self.path_plugins): + if dir_name == "Mute": + self.log.info("Deleting deprecated/renamed plugin: %s" % dir_name) + shutil.rmtree("%s/%s" % (self.path_plugins, dir_name)) + + # -- Load / Unload -- + + def listPlugins(self, list_disabled=False): + plugins = [] + for dir_name in sorted(os.listdir(self.path_plugins)): + dir_path = os.path.join(self.path_plugins, dir_name) + plugin_name = dir_name.replace("disabled-", "") + if dir_name.startswith("disabled"): + is_enabled = False + else: + is_enabled = True + + plugin_config = self.config["builtin"].get(plugin_name, {}) + if "enabled" in plugin_config: + is_enabled = plugin_config["enabled"] + + if dir_name == "__pycache__" or not os.path.isdir(dir_path): + continue # skip + if dir_name.startswith("Debug") and not config.debug: + continue # Only load in debug mode if module name starts with Debug + if not is_enabled and not list_disabled: + continue # Dont load if disabled + + plugin = {} + plugin["source"] = "builtin" + plugin["name"] = plugin_name + plugin["dir_name"] = dir_name + plugin["dir_path"] = dir_path + plugin["inner_path"] = plugin_name + plugin["enabled"] = is_enabled + plugin["rev"] = config.rev + plugin["loaded"] = plugin_name in self.plugin_names + plugins.append(plugin) + + plugins += self.listInstalledPlugins(list_disabled) + return plugins + + def listInstalledPlugins(self, list_disabled=False): + plugins = [] + + for address, site_plugins in sorted(self.config.items()): + if address == "builtin": + continue + for plugin_inner_path, plugin_config in sorted(site_plugins.items()): + is_enabled = plugin_config.get("enabled", False) + if not is_enabled and not list_disabled: + continue + plugin_name = os.path.basename(plugin_inner_path) + + dir_path = "%s/%s/%s" % (self.path_installed_plugins, address, plugin_inner_path) + + plugin = {} + plugin["source"] = address + plugin["name"] = plugin_name + plugin["dir_name"] = plugin_name + plugin["dir_path"] = dir_path + plugin["inner_path"] = plugin_inner_path + plugin["enabled"] = is_enabled + plugin["rev"] = plugin_config.get("rev", 0) + plugin["loaded"] = plugin_name in self.plugin_names + plugins.append(plugin) + + return plugins + + # Load all plugin + def loadPlugins(self): + all_loaded = True + s = time.time() + if self.path_plugins is None: + return + for plugin in self.listPlugins(): + self.log.debug("Loading plugin: %s (%s)" % (plugin["name"], plugin["source"])) + if plugin["source"] != "builtin": + self.plugins_rev[plugin["name"]] = plugin["rev"] + site_plugin_dir = os.path.dirname(plugin["dir_path"]) + if site_plugin_dir not in sys.path: + sys.path.append(site_plugin_dir) + try: + sys.modules[plugin["name"]] = __import__(plugin["dir_name"]) + except Exception as err: + self.log.error("Plugin %s load error: %s" % (plugin["name"], Debug.formatException(err))) + all_loaded = False + if plugin["name"] not in self.plugin_names: + self.plugin_names.append(plugin["name"]) + + self.log.debug("Plugins loaded in %.3fs" % (time.time() - s)) + for func in self.after_load: + func() + return all_loaded + + # Reload all plugins + def reloadPlugins(self): + self.reloading = True + self.after_load = [] + self.plugins_before = self.plugins + self.plugins = defaultdict(list) # Reset registered plugins + for module_name, module in list(sys.modules.items()): + if not module or not getattr(module, "__file__", None): + continue + if self.path_plugins not in module.__file__ and self.path_installed_plugins not in module.__file__: + continue + + if "allow_reload" in dir(module) and not module.allow_reload: # Reload disabled + # Re-add non-reloadable plugins + for class_name, classes in self.plugins_before.items(): + for c in classes: + if c.__module__ != module.__name__: + continue + self.plugins[class_name].append(c) + else: + try: + importlib.reload(module) + except Exception as err: + self.log.error("Plugin %s reload error: %s" % (module_name, Debug.formatException(err))) + + self.loadPlugins() # Load new plugins + + # Change current classes in memory + import gc + patched = {} + for class_name, classes in self.plugins.items(): + classes = classes[:] # Copy the current plugins + classes.reverse() + base_class = self.pluggable[class_name] # Original class + classes.append(base_class) # Add the class itself to end of inherience line + plugined_class = type(class_name, tuple(classes), dict()) # Create the plugined class + for obj in gc.get_objects(): + if type(obj).__name__ == class_name: + obj.__class__ = plugined_class + patched[class_name] = patched.get(class_name, 0) + 1 + self.log.debug("Patched objects: %s" % patched) + + # Change classes in modules + patched = {} + for class_name, classes in self.plugins.items(): + for module_name, module in list(sys.modules.items()): + if class_name in dir(module): + if "__class__" not in dir(getattr(module, class_name)): # Not a class + continue + base_class = self.pluggable[class_name] + classes = self.plugins[class_name][:] + classes.reverse() + classes.append(base_class) + plugined_class = type(class_name, tuple(classes), dict()) + setattr(module, class_name, plugined_class) + patched[class_name] = patched.get(class_name, 0) + 1 + + self.log.debug("Patched modules: %s" % patched) + self.reloading = False + + +plugin_manager = PluginManager() # Singletone + +# -- Decorators -- + +# Accept plugin to class decorator + + +def acceptPlugins(base_class): + class_name = base_class.__name__ + plugin_manager.pluggable[class_name] = base_class + if class_name in plugin_manager.plugins: # Has plugins + classes = plugin_manager.plugins[class_name][:] # Copy the current plugins + + # Restore the subclass order after reload + if class_name in plugin_manager.subclass_order: + classes = sorted( + classes, + key=lambda key: + plugin_manager.subclass_order[class_name].index(str(key)) + if str(key) in plugin_manager.subclass_order[class_name] + else 9999 + ) + plugin_manager.subclass_order[class_name] = list(map(str, classes)) + + classes.reverse() + classes.append(base_class) # Add the class itself to end of inherience line + plugined_class = type(class_name, tuple(classes), dict()) # Create the plugined class + plugin_manager.log.debug("New class accepts plugins: %s (Loaded plugins: %s)" % (class_name, classes)) + else: # No plugins just use the original + plugined_class = base_class + return plugined_class + + +# Register plugin to class name decorator +def registerTo(class_name): + if config.debug and not plugin_manager.reloading: + import gc + for obj in gc.get_objects(): + if type(obj).__name__ == class_name: + raise Exception("Class %s instances already present in memory" % class_name) + break + + plugin_manager.log.debug("New plugin registered to: %s" % class_name) + if class_name not in plugin_manager.plugins: + plugin_manager.plugins[class_name] = [] + + def classDecorator(self): + plugin_manager.plugins[class_name].append(self) + return self + return classDecorator + + +def afterLoad(func): + plugin_manager.after_load.append(func) + return func + + +# - Example usage - + +if __name__ == "__main__": + @registerTo("Request") + class RequestPlugin(object): + + def actionMainPage(self, path): + return "Hello MainPage!" + + @acceptPlugins + class Request(object): + + def route(self, path): + func = getattr(self, "action" + path, None) + if func: + return func(path) + else: + return "Can't route to", path + + print(Request().route("MainPage")) diff --git a/src/Plugin/__init__.py b/src/Plugin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Site/Site.py b/src/Site/Site.py new file mode 100644 index 00000000..d6179307 --- /dev/null +++ b/src/Site/Site.py @@ -0,0 +1,1147 @@ +import os +import json +import logging +import re +import time +import random +import sys +import hashlib +import collections +import base64 + +import gevent +import gevent.pool + +import util +from Config import config +from Peer import Peer +from Worker import WorkerManager +from Debug import Debug +from Content import ContentManager +from .SiteStorage import SiteStorage +from Crypt import CryptHash +from util import helper +from util import Diff +from util import GreenletManager +from Plugin import PluginManager +from File import FileServer +from .SiteAnnouncer import SiteAnnouncer +from . import SiteManager + + +@PluginManager.acceptPlugins +class Site(object): + + def __init__(self, address, allow_create=True, settings=None): + self.address = str(re.sub("[^A-Za-z0-9]", "", address)) # Make sure its correct address + self.address_hash = hashlib.sha256(self.address.encode("ascii")).digest() + self.address_sha1 = hashlib.sha1(self.address.encode("ascii")).digest() + self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging + self.log = logging.getLogger("Site:%s" % self.address_short) + self.addEventListeners() + + self.content = None # Load content.json + self.peers = {} # Key: ip:port, Value: Peer.Peer + self.peers_recent = collections.deque(maxlen=150) + self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) + self.greenlet_manager = GreenletManager.GreenletManager() # Running greenlets + self.worker_manager = WorkerManager(self) # Handle site download from other peers + self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) + self.content_updated = None # Content.js update time + self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] + self.page_requested = False # Page viewed in browser + self.websockets = [] # Active site websocket connections + + self.connection_server = None + self.loadSettings(settings) # Load settings from sites.json + self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files + self.content_manager = ContentManager(self) + self.content_manager.loadContents() # Load content.json files + if "main" in sys.modules: # import main has side-effects, breaks tests + import main + if "file_server" in dir(main): # Use global file server by default if possible + self.connection_server = main.file_server + else: + main.file_server = FileServer() + self.connection_server = main.file_server + else: + self.connection_server = FileServer() + + self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes + + if not self.settings.get("wrapper_key"): # To auth websocket permissions + self.settings["wrapper_key"] = CryptHash.random() + self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) + + if not self.settings.get("ajax_key"): # To auth websocket permissions + self.settings["ajax_key"] = CryptHash.random() + self.log.debug("New ajax key: %s" % self.settings["ajax_key"]) + + def __str__(self): + return "Site %s" % self.address_short + + def __repr__(self): + return "<%s>" % self.__str__() + + # Load site settings from data/sites.json + def loadSettings(self, settings=None): + if not settings: + settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address) + if settings: + self.settings = settings + if "cache" not in settings: + settings["cache"] = {} + if "size_files_optional" not in settings: + settings["size_optional"] = 0 + if "optional_downloaded" not in settings: + settings["optional_downloaded"] = 0 + if "downloaded" not in settings: + settings["downloaded"] = settings.get("added") + self.bad_files = settings["cache"].get("bad_files", {}) + settings["cache"]["bad_files"] = {} + # Give it minimum 10 tries after restart + for inner_path in self.bad_files: + self.bad_files[inner_path] = min(self.bad_files[inner_path], 20) + else: + self.settings = { + "own": False, "serving": True, "permissions": [], "cache": {"bad_files": {}}, "size_files_optional": 0, + "added": int(time.time()), "downloaded": None, "optional_downloaded": 0, "size_optional": 0 + } # Default + if config.download_optional == "auto": + self.settings["autodownloadoptional"] = True + + # Add admin permissions to homepage + if self.address in (config.homepage, config.updatesite) and "ADMIN" not in self.settings["permissions"]: + self.settings["permissions"].append("ADMIN") + + return + + # Save site settings to data/sites.json + def saveSettings(self): + if not SiteManager.site_manager.sites: + SiteManager.site_manager.sites = {} + if not SiteManager.site_manager.sites.get(self.address): + SiteManager.site_manager.sites[self.address] = self + SiteManager.site_manager.load(False) + SiteManager.site_manager.saveDelayed() + + def isServing(self): + if config.offline: + return False + else: + return self.settings["serving"] + + def getSettingsCache(self): + back = {} + back["bad_files"] = self.bad_files + back["hashfield"] = base64.b64encode(self.content_manager.hashfield.tobytes()).decode("ascii") + return back + + # Max site size in MB + def getSizeLimit(self): + return self.settings.get("size_limit", int(config.size_limit)) + + # Next size limit based on current size + def getNextSizeLimit(self): + size_limits = [25, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000] + size = self.settings.get("size", 0) + for size_limit in size_limits: + if size * 1.2 < size_limit * 1024 * 1024: + return size_limit + return 999999 + + def isAddedRecently(self): + return time.time() - self.settings.get("added", 0) < 60 * 60 * 24 + + # Download all file from content.json + def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}): + s = time.time() + if config.verbose: + self.log.debug( + "DownloadContent %s: Started. (download_files: %s, check_modifications: %s, diffs: %s)..." % + (inner_path, download_files, check_modifications, diffs.keys()) + ) + + if not inner_path.endswith("content.json"): + return False + + found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) + content_inner_dir = helper.getDirname(inner_path) + if not found: + self.log.debug("DownloadContent %s: Download failed, check_modifications: %s" % (inner_path, check_modifications)) + if check_modifications: # Download failed, but check modifications if its succed later + self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications") + return False # Could not download content.json + + if config.verbose: + self.log.debug("DownloadContent got %s" % inner_path) + sub_s = time.time() + + changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False) + + if config.verbose: + self.log.debug("DownloadContent %s: loadContent done in %.3fs" % (inner_path, time.time() - sub_s)) + + if inner_path == "content.json": + self.saveSettings() + + if peer: # Update last received update from peer to prevent re-sending the same update to it + peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"] + + # Verify size limit + if inner_path == "content.json": + site_size_limit = self.getSizeLimit() * 1024 * 1024 + content_size = len(json.dumps(self.content_manager.contents[inner_path], indent=1)) + sum([file["size"] for file in list(self.content_manager.contents[inner_path].get("files", {}).values()) if file["size"] >= 0]) # Size of new content + if site_size_limit < content_size: + # Not enought don't download anything + self.log.debug("DownloadContent Size limit reached (site too big please increase limit): %.2f MB > %.2f MB" % (content_size / 1024 / 1024, site_size_limit / 1024 / 1024)) + return False + + # Start download files + file_threads = [] + if download_files: + for file_relative_path in list(self.content_manager.contents[inner_path].get("files", {}).keys()): + file_inner_path = content_inner_dir + file_relative_path + + # Try to diff first + diff_success = False + diff_actions = diffs.get(file_relative_path) + if diff_actions and self.bad_files.get(file_inner_path): + try: + s = time.time() + new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions) + new_file.seek(0) + time_diff = time.time() - s + + s = time.time() + diff_success = self.content_manager.verifyFile(file_inner_path, new_file) + time_verify = time.time() - s + + if diff_success: + s = time.time() + new_file.seek(0) + self.storage.write(file_inner_path, new_file) + time_write = time.time() - s + + s = time.time() + self.onFileDone(file_inner_path) + time_on_done = time.time() - s + + self.log.debug( + "DownloadContent Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" % + (file_inner_path, time_diff, time_verify, time_write, time_on_done) + ) + except Exception as err: + self.log.debug("DownloadContent Failed to patch %s: %s" % (file_inner_path, err)) + diff_success = False + + if not diff_success: + # Start download and dont wait for finish, return the event + res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) + if res is not True and res is not False: # Need downloading and file is allowed + file_threads.append(res) # Append evt + + # Optionals files + if inner_path == "content.json": + gevent.spawn(self.updateHashfield) + + for file_relative_path in list(self.content_manager.contents[inner_path].get("files_optional", {}).keys()): + file_inner_path = content_inner_dir + file_relative_path + if file_inner_path not in changed and not self.bad_files.get(file_inner_path): + continue + if not self.isDownloadable(file_inner_path): + continue + # Start download and dont wait for finish, return the event + res = self.pooledNeedFile( + file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer + ) + if res is not True and res is not False: # Need downloading and file is allowed + file_threads.append(res) # Append evt + + # Wait for includes download + include_threads = [] + for file_relative_path in list(self.content_manager.contents[inner_path].get("includes", {}).keys()): + file_inner_path = content_inner_dir + file_relative_path + include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) + include_threads.append(include_thread) + + if config.verbose: + self.log.debug("DownloadContent %s: Downloading %s includes..." % (inner_path, len(include_threads))) + gevent.joinall(include_threads) + if config.verbose: + self.log.debug("DownloadContent %s: Includes download ended" % inner_path) + + if check_modifications: # Check if every file is up-to-date + self.checkModifications(0) + + if config.verbose: + self.log.debug("DownloadContent %s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) + gevent.joinall(file_threads) + if config.verbose: + self.log.debug("DownloadContent %s: ended in %.3fs (tasks left: %s)" % ( + inner_path, time.time() - s, len(self.worker_manager.tasks) + )) + + return True + + # Return bad files with less than 3 retry + def getReachableBadFiles(self): + if not self.bad_files: + return False + return [bad_file for bad_file, retry in self.bad_files.items() if retry < 3] + + # Retry download bad files + def retryBadFiles(self, force=False): + self.checkBadFiles() + + self.log.debug("Retry %s bad files" % len(self.bad_files)) + content_inner_paths = [] + file_inner_paths = [] + + for bad_file, tries in list(self.bad_files.items()): + if force or random.randint(0, min(40, tries)) < 4: # Larger number tries = less likely to check every 15min + if bad_file.endswith("content.json"): + content_inner_paths.append(bad_file) + else: + file_inner_paths.append(bad_file) + + if content_inner_paths: + self.pooledDownloadContent(content_inner_paths, only_if_bad=True) + + if file_inner_paths: + self.pooledDownloadFile(file_inner_paths, only_if_bad=True) + + def checkBadFiles(self): + for bad_file in list(self.bad_files.keys()): + file_info = self.content_manager.getFileInfo(bad_file) + if bad_file.endswith("content.json"): + if file_info is False and bad_file != "content.json": + del self.bad_files[bad_file] + self.log.debug("No info for file: %s, removing from bad_files" % bad_file) + else: + if file_info is False or not file_info.get("size"): + del self.bad_files[bad_file] + self.log.debug("No info or size for file: %s, removing from bad_files" % bad_file) + + # Download all files of the site + @util.Noparallel(blocking=False) + def download(self, check_size=False, blind_includes=False, retry_bad_files=True): + if not self.connection_server: + self.log.debug("No connection server found, skipping download") + return False + + s = time.time() + self.log.debug( + "Start downloading, bad_files: %s, check_size: %s, blind_includes: %s, isAddedRecently: %s" % + (self.bad_files, check_size, blind_includes, self.isAddedRecently()) + ) + + if self.isAddedRecently(): + gevent.spawn(self.announce, mode="start", force=True) + else: + gevent.spawn(self.announce, mode="update") + + if check_size: # Check the size first + valid = self.downloadContent("content.json", download_files=False) # Just download content.json files + if not valid: + return False # Cant download content.jsons or size is not fits + + # Download everything + valid = self.downloadContent("content.json", check_modifications=blind_includes) + + if retry_bad_files: + self.onComplete.once(lambda: self.retryBadFiles(force=True)) + self.log.debug("Download done in %.3fs" % (time.time() - s)) + + return valid + + def pooledDownloadContent(self, inner_paths, pool_size=100, only_if_bad=False): + self.log.debug("New downloadContent pool: len: %s, only if bad: %s" % (len(inner_paths), only_if_bad)) + self.worker_manager.started_task_num += len(inner_paths) + pool = gevent.pool.Pool(pool_size) + num_skipped = 0 + site_size_limit = self.getSizeLimit() * 1024 * 1024 + for inner_path in inner_paths: + if not only_if_bad or inner_path in self.bad_files: + pool.spawn(self.downloadContent, inner_path) + else: + num_skipped += 1 + self.worker_manager.started_task_num -= 1 + if self.settings["size"] > site_size_limit * 0.95: + self.log.warning("Site size limit almost reached, aborting downloadContent pool") + for aborted_inner_path in inner_paths: + if aborted_inner_path in self.bad_files: + del self.bad_files[aborted_inner_path] + self.worker_manager.removeSolvedFileTasks(mark_as_good=False) + break + pool.join() + self.log.debug("Ended downloadContent pool len: %s, skipped: %s" % (len(inner_paths), num_skipped)) + + def pooledDownloadFile(self, inner_paths, pool_size=100, only_if_bad=False): + self.log.debug("New downloadFile pool: len: %s, only if bad: %s" % (len(inner_paths), only_if_bad)) + self.worker_manager.started_task_num += len(inner_paths) + pool = gevent.pool.Pool(pool_size) + num_skipped = 0 + for inner_path in inner_paths: + if not only_if_bad or inner_path in self.bad_files: + pool.spawn(self.needFile, inner_path, update=True) + else: + num_skipped += 1 + self.worker_manager.started_task_num -= 1 + self.log.debug("Ended downloadFile pool len: %s, skipped: %s" % (len(inner_paths), num_skipped)) + + # Update worker, try to find client that supports listModifications command + def updater(self, peers_try, queried, since): + threads = [] + while 1: + if not peers_try or len(queried) >= 3: # Stop after 3 successful query + break + peer = peers_try.pop(0) + if config.verbose: + self.log.debug("CheckModifications: Try to get updates from: %s Left: %s" % (peer, peers_try)) + + res = None + with gevent.Timeout(20, exception=False): + res = peer.listModified(since) + + if not res or "modified_files" not in res: + continue # Failed query + + queried.append(peer) + modified_contents = [] + my_modified = self.content_manager.listModified(since) + num_old_files = 0 + for inner_path, modified in res["modified_files"].items(): # Check if the peer has newer files than we + has_newer = int(modified) > my_modified.get(inner_path, 0) + has_older = int(modified) < my_modified.get(inner_path, 0) + if inner_path not in self.bad_files and not self.content_manager.isArchived(inner_path, modified): + if has_newer: + # We dont have this file or we have older + modified_contents.append(inner_path) + self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 + if has_older and num_old_files < 5: + num_old_files += 1 + self.log.debug("CheckModifications: %s client has older version of %s, publishing there (%s/5)..." % (peer, inner_path, num_old_files)) + gevent.spawn(self.publisher, inner_path, [peer], [], 1) + if modified_contents: + self.log.debug("CheckModifications: %s new modified file from %s" % (len(modified_contents), peer)) + modified_contents.sort(key=lambda inner_path: 0 - res["modified_files"][inner_path]) # Download newest first + t = gevent.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True) + threads.append(t) + if config.verbose: + self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads)) + gevent.joinall(threads) + + # Check modified content.json files from peers and add modified files to bad_files + # Return: Successfully queried peers [Peer, Peer...] + def checkModifications(self, since=None): + s = time.time() + peers_try = [] # Try these peers + queried = [] # Successfully queried from these peers + limit = 5 + + # Wait for peers + if not self.peers: + self.announce(mode="update") + for wait in range(10): + time.sleep(5 + wait) + self.log.debug("CheckModifications: Waiting for peers...") + if self.peers: + break + + peers_try = self.getConnectedPeers() + peers_connected_num = len(peers_try) + if peers_connected_num < limit * 2: # Add more, non-connected peers if necessary + peers_try += self.getRecentPeers(limit * 5) + + if since is None: # No since defined, download from last modification time-1day + since = self.settings.get("modified", 60 * 60 * 24) - 60 * 60 * 24 + + if config.verbose: + self.log.debug( + "CheckModifications: Try to get listModifications from peers: %s, connected: %s, since: %s" % + (peers_try, peers_connected_num, since) + ) + + updaters = [] + for i in range(3): + updaters.append(gevent.spawn(self.updater, peers_try, queried, since)) + + gevent.joinall(updaters, timeout=10) # Wait 10 sec to workers done query modifications + + if not queried: # Start another 3 thread if first 3 is stuck + peers_try[0:0] = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Add connected peers + for _ in range(10): + gevent.joinall(updaters, timeout=10) # Wait another 10 sec if none of updaters finished + if queried: + break + + self.log.debug("CheckModifications: Queried listModifications from: %s in %.3fs since %s" % (queried, time.time() - s, since)) + time.sleep(0.1) + return queried + + # Update content.json from peers and download changed files + # Return: None + @util.Noparallel() + def update(self, announce=False, check_files=False, since=None): + self.content_manager.loadContent("content.json", load_includes=False) # Reload content.json + self.content_updated = None # Reset content updated time + + if check_files: + self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size + + if not self.isServing(): + return False + + self.updateWebsocket(updating=True) + + # Remove files that no longer in content.json + self.checkBadFiles() + + if announce: + self.announce(mode="update", force=True) + + # Full update, we can reset bad files + if check_files and since == 0: + self.bad_files = {} + + queried = self.checkModifications(since) + + changed, deleted = self.content_manager.loadContent("content.json", load_includes=False) + + if self.bad_files: + self.log.debug("Bad files: %s" % self.bad_files) + gevent.spawn(self.retryBadFiles, force=True) + + if len(queried) == 0: + # Failed to query modifications + self.content_updated = False + else: + self.content_updated = time.time() + + self.updateWebsocket(updated=True) + + # Update site by redownload all content.json + def redownloadContents(self): + # Download all content.json again + content_threads = [] + for inner_path in list(self.content_manager.contents.keys()): + content_threads.append(self.needFile(inner_path, update=True, blocking=False)) + + self.log.debug("Waiting %s content.json to finish..." % len(content_threads)) + gevent.joinall(content_threads) + + # Publish worker + def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None): + file_size = self.storage.getSize(inner_path) + content_json_modified = self.content_manager.contents[inner_path]["modified"] + body = self.storage.read(inner_path) + + while 1: + if not peers or len(published) >= limit: + if event_done: + event_done.set(True) + break # All peers done, or published engouht + peer = peers.pop() + if peer in published: + continue + if peer.last_content_json_update == content_json_modified: + self.log.debug("%s already received this update for %s, skipping" % (peer, inner_path)) + continue + + if peer.connection and peer.connection.last_ping_delay: # Peer connected + # Timeout: 5sec + size in kb + last_ping + timeout = 5 + int(file_size / 1024) + peer.connection.last_ping_delay + else: # Peer not connected + # Timeout: 10sec + size in kb + timeout = 10 + int(file_size / 1024) + result = {"exception": "Timeout"} + + for retry in range(2): + try: + with gevent.Timeout(timeout, False): + result = peer.publish(self.address, inner_path, body, content_json_modified, diffs) + if result: + break + except Exception as err: + self.log.error("Publish error: %s" % Debug.formatException(err)) + result = {"exception": Debug.formatException(err)} + + if result and "ok" in result: + published.append(peer) + if cb_progress and len(published) <= limit: + cb_progress(len(published), limit) + self.log.info("[OK] %s: %s %s/%s" % (peer.key, result["ok"], len(published), limit)) + else: + if result == {"exception": "Timeout"}: + peer.onConnectionError("Publish timeout") + self.log.info("[FAILED] %s: %s" % (peer.key, result)) + time.sleep(0.01) + + # Update content.json on peers + @util.Noparallel() + def publish(self, limit="default", inner_path="content.json", diffs={}, cb_progress=None): + published = [] # Successfully published (Peer) + publishers = [] # Publisher threads + + if not self.peers: + self.announce(mode="more") + + if limit == "default": + limit = 5 + threads = limit + + peers = self.getConnectedPeers() + num_connected_peers = len(peers) + + random.shuffle(peers) + peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) # Prefer newer clients + + if len(peers) < limit * 2 and len(self.peers) > len(peers): # Add more, non-connected peers if necessary + peers += self.getRecentPeers(limit * 2) + + peers = set(peers) + + self.log.info("Publishing %s to %s/%s peers (connected: %s) diffs: %s (%.2fk)..." % ( + inner_path, limit, len(self.peers), num_connected_peers, list(diffs.keys()), float(len(str(diffs))) / 1024 + )) + + if not peers: + return 0 # No peers found + + event_done = gevent.event.AsyncResult() + for i in range(min(len(peers), limit, threads)): + publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, diffs, event_done, cb_progress) + publishers.append(publisher) + + event_done.get() # Wait for done + if len(published) < min(len(self.peers), limit): + time.sleep(0.2) # If less than we need sleep a bit + if len(published) == 0: + gevent.joinall(publishers) # No successful publish, wait for all publisher + + # Publish more peers in the backgroup + self.log.info( + "Published %s to %s peers, publishing to %s more peers in the background" % + (inner_path, len(published), limit) + ) + + for thread in range(2): + gevent.spawn(self.publisher, inner_path, peers, published, limit=limit * 2, diffs=diffs) + + # Send my hashfield to every connected peer if changed + gevent.spawn(self.sendMyHashfield, 100) + + return len(published) + + # Copy this site + @util.Noparallel() + def clone(self, address, privatekey=None, address_index=None, root_inner_path="", overwrite=False): + import shutil + new_site = SiteManager.site_manager.need(address, all_file=False) + default_dirs = [] # Dont copy these directories (has -default version) + for dir_name in os.listdir(self.storage.directory): + if "-default" in dir_name: + default_dirs.append(dir_name.replace("-default", "")) + + self.log.debug("Cloning to %s, ignore dirs: %s, root: %s" % (address, default_dirs, root_inner_path)) + + # Copy root content.json + if not new_site.storage.isFile("content.json") and not overwrite: + # New site: Content.json not exist yet, create a new one from source site + if "size_limit" in self.settings: + new_site.settings["size_limit"] = self.settings["size_limit"] + + # Use content.json-default is specified + if self.storage.isFile(root_inner_path + "/content.json-default"): + content_json = self.storage.loadJson(root_inner_path + "/content.json-default") + else: + content_json = self.storage.loadJson("content.json") + + if "domain" in content_json: + del content_json["domain"] + content_json["title"] = "my" + content_json["title"] + content_json["cloned_from"] = self.address + content_json["clone_root"] = root_inner_path + content_json["files"] = {} + if address_index: + content_json["address_index"] = address_index # Site owner's BIP32 index + new_site.storage.writeJson("content.json", content_json) + new_site.content_manager.loadContent( + "content.json", add_bad_files=False, delete_removed_files=False, load_includes=False + ) + + # Copy files + for content_inner_path, content in list(self.content_manager.contents.items()): + file_relative_paths = list(content.get("files", {}).keys()) + + # Sign content.json at the end to make sure every file is included + file_relative_paths.sort() + file_relative_paths.sort(key=lambda key: key.replace("-default", "").endswith("content.json")) + + for file_relative_path in file_relative_paths: + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to content.json + file_inner_path = file_inner_path.strip("/") # Strip leading / + if not file_inner_path.startswith(root_inner_path): + self.log.debug("[SKIP] %s (not in clone root)" % file_inner_path) + continue + if file_inner_path.split("/")[0] in default_dirs: # Dont copy directories that has -default postfixed alternative + self.log.debug("[SKIP] %s (has default alternative)" % file_inner_path) + continue + file_path = self.storage.getPath(file_inner_path) + + # Copy the file normally to keep the -default postfixed dir and file to allow cloning later + if root_inner_path: + file_inner_path_dest = re.sub("^%s/" % re.escape(root_inner_path), "", file_inner_path) + file_path_dest = new_site.storage.getPath(file_inner_path_dest) + else: + file_inner_path_dest = file_inner_path + file_path_dest = new_site.storage.getPath(file_inner_path) + + self.log.debug("[COPY] %s to %s..." % (file_inner_path, file_path_dest)) + dest_dir = os.path.dirname(file_path_dest) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + if file_inner_path_dest.replace("-default", "") == "content.json": # Don't copy root content.json-default + continue + + shutil.copy(file_path, file_path_dest) + + # If -default in path, create a -default less copy of the file + if "-default" in file_inner_path_dest: + file_path_dest = new_site.storage.getPath(file_inner_path_dest.replace("-default", "")) + if new_site.storage.isFile(file_inner_path_dest.replace("-default", "")) and not overwrite: + # Don't overwrite site files with default ones + self.log.debug("[SKIP] Default file: %s (already exist)" % file_inner_path) + continue + self.log.debug("[COPY] Default file: %s to %s..." % (file_inner_path, file_path_dest)) + dest_dir = os.path.dirname(file_path_dest) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + shutil.copy(file_path, file_path_dest) + # Sign if content json + if file_path_dest.endswith("/content.json"): + new_site.storage.onUpdated(file_inner_path_dest.replace("-default", "")) + new_site.content_manager.loadContent( + file_inner_path_dest.replace("-default", ""), add_bad_files=False, + delete_removed_files=False, load_includes=False + ) + if privatekey: + new_site.content_manager.sign(file_inner_path_dest.replace("-default", ""), privatekey, remove_missing_optional=True) + new_site.content_manager.loadContent( + file_inner_path_dest, add_bad_files=False, delete_removed_files=False, load_includes=False + ) + + if privatekey: + new_site.content_manager.sign("content.json", privatekey, remove_missing_optional=True) + new_site.content_manager.loadContent( + "content.json", add_bad_files=False, delete_removed_files=False, load_includes=False + ) + + # Rebuild DB + if new_site.storage.isFile("dbschema.json"): + new_site.storage.closeDb() + try: + new_site.storage.rebuildDb() + except Exception as err: + self.log.error(err) + + return new_site + + @util.Pooled(100) + def pooledNeedFile(self, *args, **kwargs): + return self.needFile(*args, **kwargs) + + def isFileDownloadAllowed(self, inner_path, file_info): + # Verify space for all site + if self.settings["size"] > self.getSizeLimit() * 1024 * 1024: + return False + # Verify space for file + if file_info.get("size", 0) > config.file_size_limit * 1024 * 1024: + self.log.debug( + "File size %s too large: %sMB > %sMB, skipping..." % + (inner_path, file_info.get("size", 0) / 1024 / 1024, config.file_size_limit) + ) + return False + else: + return True + + def needFileInfo(self, inner_path): + file_info = self.content_manager.getFileInfo(inner_path) + if not file_info: + # No info for file, download all content.json first + self.log.debug("No info for %s, waiting for all content.json" % inner_path) + success = self.downloadContent("content.json", download_files=False) + if not success: + return False + file_info = self.content_manager.getFileInfo(inner_path) + return file_info + + # Check and download if file not exist + def needFile(self, inner_path, update=False, blocking=True, peer=None, priority=0): + if self.worker_manager.tasks.findTask(inner_path): + task = self.worker_manager.addTask(inner_path, peer, priority=priority) + if blocking: + return task["evt"].get() + else: + return task["evt"] + elif self.storage.isFile(inner_path) and not update: # File exist, no need to do anything + return True + elif not self.isServing(): # Site not serving + return False + else: # Wait until file downloaded + if not self.content_manager.contents.get("content.json"): # No content.json, download it first! + self.log.debug("Need content.json first (inner_path: %s, priority: %s)" % (inner_path, priority)) + if priority > 0: + gevent.spawn(self.announce) + if inner_path != "content.json": # Prevent double download + task = self.worker_manager.addTask("content.json", peer) + task["evt"].get() + self.content_manager.loadContent() + if not self.content_manager.contents.get("content.json"): + return False # Content.json download failed + + file_info = None + if not inner_path.endswith("content.json"): + file_info = self.needFileInfo(inner_path) + if not file_info: + return False + if "cert_signers" in file_info and not file_info["content_inner_path"] in self.content_manager.contents: + self.log.debug("Missing content.json for requested user file: %s" % inner_path) + if self.bad_files.get(file_info["content_inner_path"], 0) > 5: + self.log.debug("File %s not reachable: retry %s" % ( + inner_path, self.bad_files.get(file_info["content_inner_path"], 0) + )) + return False + self.downloadContent(file_info["content_inner_path"]) + + if not self.isFileDownloadAllowed(inner_path, file_info): + self.log.debug("%s: Download not allowed" % inner_path) + return False + + self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file + + task = self.worker_manager.addTask(inner_path, peer, priority=priority, file_info=file_info) + if blocking: + return task["evt"].get() + else: + return task["evt"] + + # Add or update a peer to site + # return_peer: Always return the peer even if it was already present + def addPeer(self, ip, port, return_peer=False, connection=None, source="other"): + if not ip or ip == "0.0.0.0": + return False + + key = "%s:%s" % (ip, port) + peer = self.peers.get(key) + if peer: # Already has this ip + peer.found(source) + if return_peer: # Always return peer + return peer + else: + return False + else: # New peer + if (ip, port) in self.peer_blacklist: + return False # Ignore blacklist (eg. myself) + peer = Peer(ip, port, self) + self.peers[key] = peer + peer.found(source) + return peer + + def announce(self, *args, **kwargs): + if self.isServing(): + self.announcer.announce(*args, **kwargs) + + # Keep connections to get the updates + def needConnections(self, num=None, check_site_on_reconnect=False): + if num is None: + if len(self.peers) < 50: + num = 3 + else: + num = 6 + need = min(len(self.peers), num, config.connected_limit) # Need 5 peer, but max total peers + + connected = len(self.getConnectedPeers()) + + connected_before = connected + + self.log.debug("Need connections: %s, Current: %s, Total: %s" % (need, connected, len(self.peers))) + + if connected < need: # Need more than we have + for peer in self.getRecentPeers(30): + if not peer.connection or not peer.connection.connected: # No peer connection or disconnected + peer.pex() # Initiate peer exchange + if peer.connection and peer.connection.connected: + connected += 1 # Successfully connected + if connected >= need: + break + self.log.debug( + "Connected before: %s, after: %s. Check site: %s." % + (connected_before, connected, check_site_on_reconnect) + ) + + if check_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet: + gevent.spawn(self.update, check_files=False) + + return connected + + # Return: Probably peers verified to be connectable recently + def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True): + peers = list(self.peers.values()) + found = [] + for peer in peers: + if peer.key.endswith(":0"): + continue # Not connectable + if not peer.connection: + continue # No connection + if peer.ip.endswith(".onion") and not self.connection_server.tor_manager.enabled: + continue # Onion not supported + if peer.key in ignore: + continue # The requester has this peer + if time.time() - peer.connection.last_recv_time > 60 * 60 * 2: # Last message more than 2 hours ago + peer.connection = None # Cleanup: Dead connection + continue + if not allow_private and helper.isPrivateIp(peer.ip): + continue + found.append(peer) + if len(found) >= need_num: + break # Found requested number of peers + + if len(found) < need_num: # Return not that good peers + found += [ + peer for peer in peers + if not peer.key.endswith(":0") and + peer.key not in ignore and + (allow_private or not helper.isPrivateIp(peer.ip)) + ][0:need_num - len(found)] + + return found + + # Return: Recently found peers + def getRecentPeers(self, need_num): + found = list(set(self.peers_recent)) + self.log.debug( + "Recent peers %s of %s (need: %s)" % + (len(found), len(self.peers), need_num) + ) + + if len(found) >= need_num or len(found) >= len(self.peers): + return sorted( + found, + key=lambda peer: peer.reputation, + reverse=True + )[0:need_num] + + # Add random peers + need_more = need_num - len(found) + if not self.connection_server.tor_manager.enabled: + peers = [peer for peer in self.peers.values() if not peer.ip.endswith(".onion")] + else: + peers = list(self.peers.values()) + + found_more = sorted( + peers[0:need_more * 50], + key=lambda peer: peer.reputation, + reverse=True + )[0:need_more * 2] + + found += found_more + + return found[0:need_num] + + def getConnectedPeers(self): + back = [] + if not self.connection_server: + return [] + + tor_manager = self.connection_server.tor_manager + for connection in self.connection_server.connections: + if not connection.connected and time.time() - connection.start_time > 20: # Still not connected after 20s + continue + peer = self.peers.get("%s:%s" % (connection.ip, connection.port)) + if peer: + if connection.ip.endswith(".onion") and connection.target_onion and tor_manager.start_onions: + # Check if the connection is made with the onion address created for the site + valid_target_onions = (tor_manager.getOnion(self.address), tor_manager.getOnion("global")) + if connection.target_onion not in valid_target_onions: + continue + if not peer.connection: + peer.connect(connection) + back.append(peer) + return back + + # Cleanup probably dead peers and close connection if too much + def cleanupPeers(self, peers_protected=[]): + peers = list(self.peers.values()) + if len(peers) > 20: + # Cleanup old peers + removed = 0 + if len(peers) > 1000: + ttl = 60 * 60 * 1 + else: + ttl = 60 * 60 * 4 + + for peer in peers: + if peer.connection and peer.connection.connected: + continue + if peer.connection and not peer.connection.connected: + peer.connection = None # Dead connection + if time.time() - peer.time_found > ttl: # Not found on tracker or via pex in last 4 hour + peer.remove("Time found expired") + removed += 1 + if removed > len(peers) * 0.1: # Don't remove too much at once + break + + if removed: + self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers))) + + # Close peers over the limit + closed = 0 + connected_peers = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Only fully connected peers + need_to_close = len(connected_peers) - config.connected_limit + + if closed < need_to_close: + # Try to keep connections with more sites + for peer in sorted(connected_peers, key=lambda peer: min(peer.connection.sites, 5)): + if not peer.connection: + continue + if peer.key in peers_protected: + continue + if peer.connection.sites > 5: + break + peer.connection.close("Cleanup peers") + peer.connection = None + closed += 1 + if closed >= need_to_close: + break + + if need_to_close > 0: + self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (len(connected_peers), need_to_close, closed)) + + # Send hashfield to peers + def sendMyHashfield(self, limit=5): + if not self.content_manager.hashfield: # No optional files + return False + + sent = 0 + connected_peers = self.getConnectedPeers() + for peer in connected_peers: + if peer.sendMyHashfield(): + sent += 1 + if sent >= limit: + break + if sent: + my_hashfield_changed = self.content_manager.hashfield.time_changed + self.log.debug("Sent my hashfield (chaged %.3fs ago) to %s peers" % (time.time() - my_hashfield_changed, sent)) + return sent + + # Update hashfield + def updateHashfield(self, limit=5): + # Return if no optional files + if not self.content_manager.hashfield and not self.content_manager.has_optional_files: + return False + + s = time.time() + queried = 0 + connected_peers = self.getConnectedPeers() + for peer in connected_peers: + if peer.time_hashfield: + continue + if peer.updateHashfield(): + queried += 1 + if queried >= limit: + break + if queried: + self.log.debug("Queried hashfield from %s peers in %.3fs" % (queried, time.time() - s)) + return queried + + # Returns if the optional file is need to be downloaded or not + def isDownloadable(self, inner_path): + return self.settings.get("autodownloadoptional") + + def delete(self): + self.log.info("Deleting site...") + s = time.time() + self.settings["serving"] = False + self.settings["deleting"] = True + self.saveSettings() + num_greenlets = self.greenlet_manager.stopGreenlets("Site %s deleted" % self.address) + self.worker_manager.running = False + num_workers = self.worker_manager.stopWorkers() + SiteManager.site_manager.delete(self.address) + self.content_manager.contents.db.deleteSite(self) + self.updateWebsocket(deleted=True) + self.storage.deleteFiles() + self.log.info( + "Deleted site in %.3fs (greenlets: %s, workers: %s)" % + (time.time() - s, num_greenlets, num_workers) + ) + + # - Events - + + # Add event listeners + def addEventListeners(self): + self.onFileStart = util.Event() # If WorkerManager added new task + self.onFileDone = util.Event() # If WorkerManager successfully downloaded a file + self.onFileFail = util.Event() # If WorkerManager failed to download a file + self.onComplete = util.Event() # All file finished + + self.onFileStart.append(lambda inner_path: self.fileStarted()) # No parameters to make Noparallel batching working + self.onFileDone.append(lambda inner_path: self.fileDone(inner_path)) + self.onFileFail.append(lambda inner_path: self.fileFailed(inner_path)) + + # Send site status update to websocket clients + def updateWebsocket(self, **kwargs): + if kwargs: + param = {"event": list(kwargs.items())[0]} + else: + param = None + for ws in self.websockets: + ws.event("siteChanged", self, param) + + def messageWebsocket(self, message, type="info", progress=None): + for ws in self.websockets: + if progress is None: + ws.cmd("notification", [type, message]) + else: + ws.cmd("progress", [type, message, progress]) + + # File download started + @util.Noparallel(blocking=False) + def fileStarted(self): + time.sleep(0.001) # Wait for other files adds + self.updateWebsocket(file_started=True) + + # File downloaded successful + def fileDone(self, inner_path): + # File downloaded, remove it from bad files + if inner_path in self.bad_files: + if config.verbose: + self.log.debug("Bad file solved: %s" % inner_path) + del(self.bad_files[inner_path]) + + # Update content.json last downlad time + if inner_path == "content.json": + if not self.settings.get("downloaded"): + self.settings["downloaded"] = int(time.time()) + self.content_updated = time.time() + + self.updateWebsocket(file_done=inner_path) + + # File download failed + def fileFailed(self, inner_path): + if inner_path == "content.json": + self.content_updated = False + self.log.debug("Can't update content.json") + if inner_path in self.bad_files and self.connection_server.has_internet: + self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 + + self.updateWebsocket(file_failed=inner_path) + + if self.bad_files.get(inner_path, 0) > 30: + self.fileForgot(inner_path) + + def fileForgot(self, inner_path): + self.log.debug("Giving up on %s" % inner_path) + del self.bad_files[inner_path] # Give up after 30 tries diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py new file mode 100644 index 00000000..2fd63e82 --- /dev/null +++ b/src/Site/SiteAnnouncer.py @@ -0,0 +1,293 @@ +import random +import time +import hashlib +import re +import collections + +import gevent + +from Plugin import PluginManager +from Config import config +from Debug import Debug +from util import helper +from greenlet import GreenletExit +import util + + +class AnnounceError(Exception): + pass + +global_stats = collections.defaultdict(lambda: collections.defaultdict(int)) + + +@PluginManager.acceptPlugins +class SiteAnnouncer(object): + def __init__(self, site): + self.site = site + self.stats = {} + self.fileserver_port = config.fileserver_port + self.peer_id = self.site.connection_server.peer_id + self.last_tracker_id = random.randint(0, 10) + self.time_last_announce = 0 + + def getTrackers(self): + return config.trackers + + def getSupportedTrackers(self): + trackers = self.getTrackers() + + if not self.site.connection_server.tor_manager.enabled: + trackers = [tracker for tracker in trackers if ".onion" not in tracker] + + trackers = [tracker for tracker in trackers if self.getAddressParts(tracker)] # Remove trackers with unknown address + + if "ipv6" not in self.site.connection_server.supported_ip_types: + trackers = [tracker for tracker in trackers if helper.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"] + + return trackers + + def getAnnouncingTrackers(self, mode): + trackers = self.getSupportedTrackers() + + if trackers and (mode == "update" or mode == "more"): # Only announce on one tracker, increment the queried tracker id + self.last_tracker_id += 1 + self.last_tracker_id = self.last_tracker_id % len(trackers) + trackers_announcing = [trackers[self.last_tracker_id]] # We only going to use this one + else: + trackers_announcing = trackers + + return trackers_announcing + + def getOpenedServiceTypes(self): + back = [] + # Type of addresses they can reach me + if config.trackers_proxy == "disable" and config.tor != "always": + for ip_type, opened in list(self.site.connection_server.port_opened.items()): + if opened: + back.append(ip_type) + if self.site.connection_server.tor_manager.start_onions: + back.append("onion") + return back + + @util.Noparallel(blocking=False) + def announce(self, force=False, mode="start", pex=True): + if time.time() - self.time_last_announce < 30 and not force: + return # No reannouncing within 30 secs + if force: + self.site.log.debug("Force reannounce in mode %s" % mode) + + self.fileserver_port = config.fileserver_port + self.time_last_announce = time.time() + + trackers = self.getAnnouncingTrackers(mode) + + if config.verbose: + self.site.log.debug("Tracker announcing, trackers: %s" % trackers) + + errors = [] + slow = [] + s = time.time() + threads = [] + num_announced = 0 + + for tracker in trackers: # Start announce threads + tracker_stats = global_stats[tracker] + # Reduce the announce time for trackers that looks unreliable + time_announce_allowed = time.time() - 60 * min(30, tracker_stats["num_error"]) + if tracker_stats["num_error"] > 5 and tracker_stats["time_request"] > time_announce_allowed and not force: + if config.verbose: + self.site.log.debug("Tracker %s looks unreliable, announce skipped (error: %s)" % (tracker, tracker_stats["num_error"])) + continue + thread = self.site.greenlet_manager.spawn(self.announceTracker, tracker, mode=mode) + threads.append(thread) + thread.tracker = tracker + + time.sleep(0.01) + self.updateWebsocket(trackers="announcing") + + gevent.joinall(threads, timeout=20) # Wait for announce finish + + for thread in threads: + if thread.value is None: + continue + if thread.value is not False: + if thread.value > 1.0: # Takes more than 1 second to announce + slow.append("%.2fs %s" % (thread.value, thread.tracker)) + num_announced += 1 + else: + if thread.ready(): + errors.append(thread.tracker) + else: # Still running + slow.append("30s+ %s" % thread.tracker) + + # Save peers num + self.site.settings["peers"] = len(self.site.peers) + + if len(errors) < len(threads): # At least one tracker finished + if len(trackers) == 1: + announced_to = trackers[0] + else: + announced_to = "%s/%s trackers" % (num_announced, len(threads)) + if mode != "update" or config.verbose: + self.site.log.debug( + "Announced in mode %s to %s in %.3fs, errors: %s, slow: %s" % + (mode, announced_to, time.time() - s, errors, slow) + ) + else: + if len(threads) > 1: + self.site.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s)) + if len(threads) == 1 and mode != "start": # Move to next tracker + self.site.log.debug("Tracker failed, skipping to next one...") + self.site.greenlet_manager.spawnLater(1.0, self.announce, force=force, mode=mode, pex=pex) + + self.updateWebsocket(trackers="announced") + + if pex: + self.updateWebsocket(pex="announcing") + if mode == "more": # Need more peers + self.announcePex(need_num=10) + else: + self.announcePex() + + self.updateWebsocket(pex="announced") + + def getTrackerHandler(self, protocol): + return None + + def getAddressParts(self, tracker): + if "://" not in tracker or not re.match("^[A-Za-z0-9:/\\.#-]+$", tracker): + return None + protocol, address = tracker.split("://", 1) + if ":" in address: + ip, port = address.rsplit(":", 1) + else: + ip = address + if protocol.startswith("https"): + port = 443 + else: + port = 80 + back = {} + back["protocol"] = protocol + back["address"] = address + back["ip"] = ip + back["port"] = port + return back + + def announceTracker(self, tracker, mode="start", num_want=10): + s = time.time() + address_parts = self.getAddressParts(tracker) + if not address_parts: + self.site.log.warning("Tracker %s error: Invalid address" % tracker) + return False + + if tracker not in self.stats: + self.stats[tracker] = {"status": "", "num_request": 0, "num_success": 0, "num_error": 0, "time_request": 0, "time_last_error": 0} + + last_status = self.stats[tracker]["status"] + self.stats[tracker]["status"] = "announcing" + self.stats[tracker]["time_request"] = time.time() + global_stats[tracker]["time_request"] = time.time() + if config.verbose: + self.site.log.debug("Tracker announcing to %s (mode: %s)" % (tracker, mode)) + if mode == "update": + num_want = 10 + else: + num_want = 30 + + handler = self.getTrackerHandler(address_parts["protocol"]) + error = None + try: + if handler: + peers = handler(address_parts["address"], mode=mode, num_want=num_want) + else: + raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"]) + except Exception as err: + self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) + error = err + + if error: + self.stats[tracker]["status"] = "error" + self.stats[tracker]["time_status"] = time.time() + self.stats[tracker]["last_error"] = str(error) + self.stats[tracker]["time_last_error"] = time.time() + if self.site.connection_server.has_internet: + self.stats[tracker]["num_error"] += 1 + self.stats[tracker]["num_request"] += 1 + global_stats[tracker]["num_request"] += 1 + if self.site.connection_server.has_internet: + global_stats[tracker]["num_error"] += 1 + self.updateWebsocket(tracker="error") + return False + + if peers is None: # Announce skipped + self.stats[tracker]["time_status"] = time.time() + self.stats[tracker]["status"] = last_status + return None + + self.stats[tracker]["status"] = "announced" + self.stats[tracker]["time_status"] = time.time() + self.stats[tracker]["num_success"] += 1 + self.stats[tracker]["num_request"] += 1 + global_stats[tracker]["num_request"] += 1 + global_stats[tracker]["num_error"] = 0 + + if peers is True: # Announce success, but no peers returned + return time.time() - s + + # Adding peers + added = 0 + for peer in peers: + if peer["port"] == 1: # Some trackers does not accept port 0, so we send port 1 as not-connectable + peer["port"] = 0 + if not peer["port"]: + continue # Dont add peers with port 0 + if self.site.addPeer(peer["addr"], peer["port"], source="tracker"): + added += 1 + + if added: + self.site.worker_manager.onPeers() + self.site.updateWebsocket(peers_added=added) + + if config.verbose: + self.site.log.debug( + "Tracker result: %s://%s (found %s peers, new: %s, total: %s)" % + (address_parts["protocol"], address_parts["address"], len(peers), added, len(self.site.peers)) + ) + return time.time() - s + + @util.Noparallel(blocking=False) + def announcePex(self, query_num=2, need_num=5): + peers = self.site.getConnectedPeers() + if len(peers) == 0: # Wait 3s for connections + time.sleep(3) + peers = self.site.getConnectedPeers() + + if len(peers) == 0: # Small number of connected peers for this site, connect to any + peers = list(self.site.getRecentPeers(20)) + need_num = 10 + + random.shuffle(peers) + done = 0 + total_added = 0 + for peer in peers: + num_added = peer.pex(need_num=need_num) + if num_added is not False: + done += 1 + total_added += num_added + if num_added: + self.site.worker_manager.onPeers() + self.site.updateWebsocket(peers_added=num_added) + else: + time.sleep(0.1) + if done == query_num: + break + self.site.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added)) + + def updateWebsocket(self, **kwargs): + if kwargs: + param = {"event": list(kwargs.items())[0]} + else: + param = None + + for ws in self.site.websockets: + ws.event("announcerChanged", self.site, param) diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py new file mode 100644 index 00000000..684d69fc --- /dev/null +++ b/src/Site/SiteManager.py @@ -0,0 +1,226 @@ +import json +import logging +import re +import os +import time +import atexit + +import gevent + +import util +from Plugin import PluginManager +from Content import ContentDb +from Config import config +from util import helper +from util import RateLimit +from util import Cached + + +@PluginManager.acceptPlugins +class SiteManager(object): + def __init__(self): + self.log = logging.getLogger("SiteManager") + self.log.debug("SiteManager created.") + self.sites = {} + self.sites_changed = int(time.time()) + self.loaded = False + gevent.spawn(self.saveTimer) + atexit.register(lambda: self.save(recalculate_size=True)) + + # Load all sites from data/sites.json + @util.Noparallel() + def load(self, cleanup=True, startup=False): + from Debug import Debug + self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup)) + self.loaded = False + from .Site import Site + address_found = [] + added = 0 + load_s = time.time() + # Load new adresses + try: + json_path = "%s/sites.json" % config.data_dir + data = json.load(open(json_path)) + except Exception as err: + raise Exception("Unable to load %s: %s" % (json_path, err)) + + sites_need = [] + + for address, settings in data.items(): + if address not in self.sites: + if os.path.isfile("%s/%s/content.json" % (config.data_dir, address)): + # Root content.json exists, try load site + s = time.time() + try: + site = Site(address, settings=settings) + site.content_manager.contents.get("content.json") + except Exception as err: + self.log.debug("Error loading site %s: %s" % (address, err)) + continue + self.sites[address] = site + self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s)) + added += 1 + elif startup: + # No site directory, start download + self.log.debug("Found new site in sites.json: %s" % address) + sites_need.append([address, settings]) + added += 1 + + address_found.append(address) + + # Remove deleted adresses + if cleanup: + for address in list(self.sites.keys()): + if address not in address_found: + del(self.sites[address]) + self.log.debug("Removed site: %s" % address) + + # Remove orpan sites from contentdb + content_db = ContentDb.getContentDb() + for row in content_db.execute("SELECT * FROM site").fetchall(): + address = row["address"] + if address not in self.sites and address not in address_found: + self.log.info("Deleting orphan site from content.db: %s" % address) + + try: + content_db.execute("DELETE FROM site WHERE ?", {"address": address}) + except Exception as err: + self.log.error("Can't delete site %s from content_db: %s" % (address, err)) + + if address in content_db.site_ids: + del content_db.site_ids[address] + if address in content_db.sites: + del content_db.sites[address] + + self.loaded = True + for address, settings in sites_need: + gevent.spawn(self.need, address, settings=settings) + if added: + self.log.info("Added %s sites in %.3fs" % (added, time.time() - load_s)) + + def saveDelayed(self): + RateLimit.callAsync("Save sites.json", allowed_again=5, func=self.save) + + def save(self, recalculate_size=False): + if not self.sites: + self.log.debug("Save skipped: No sites found") + return + if not self.loaded: + self.log.debug("Save skipped: Not loaded") + return + s = time.time() + data = {} + # Generate data file + s = time.time() + for address, site in list(self.list().items()): + if recalculate_size: + site.settings["size"], site.settings["size_optional"] = site.content_manager.getTotalSize() # Update site size + data[address] = site.settings + data[address]["cache"] = site.getSettingsCache() + time_generate = time.time() - s + + s = time.time() + if data: + helper.atomicWrite("%s/sites.json" % config.data_dir, helper.jsonDumps(data).encode("utf8")) + else: + self.log.debug("Save error: No data") + time_write = time.time() - s + + # Remove cache from site settings + for address, site in self.list().items(): + site.settings["cache"] = {} + + self.log.debug("Saved sites in %.2fs (generate: %.2fs, write: %.2fs)" % (time.time() - s, time_generate, time_write)) + + def saveTimer(self): + while 1: + time.sleep(60 * 10) + self.save(recalculate_size=True) + + # Checks if its a valid address + def isAddress(self, address): + return re.match("^[A-Za-z0-9]{26,35}$", address) + + def isDomain(self, address): + return False + + @Cached(timeout=10) + def isDomainCached(self, address): + return self.isDomain(address) + + def resolveDomain(self, domain): + return False + + @Cached(timeout=10) + def resolveDomainCached(self, domain): + return self.resolveDomain(domain) + + # Return: Site object or None if not found + def get(self, address): + if self.isDomainCached(address): + address_resolved = self.resolveDomainCached(address) + if address_resolved: + address = address_resolved + + if not self.loaded: # Not loaded yet + self.log.debug("Loading site: %s)..." % address) + self.load() + site = self.sites.get(address) + + return site + + def add(self, address, all_file=True, settings=None, **kwargs): + from .Site import Site + self.sites_changed = int(time.time()) + # Try to find site with differect case + for recover_address, recover_site in list(self.sites.items()): + if recover_address.lower() == address.lower(): + return recover_site + + if not self.isAddress(address): + return False # Not address: %s % address + self.log.debug("Added new site: %s" % address) + config.loadTrackersFile() + site = Site(address, settings=settings) + self.sites[address] = site + if not site.settings["serving"]: # Maybe it was deleted before + site.settings["serving"] = True + site.saveSettings() + if all_file: # Also download user files on first sync + site.download(check_size=True, blind_includes=True) + return site + + # Return or create site and start download site files + def need(self, address, *args, **kwargs): + if self.isDomainCached(address): + address_resolved = self.resolveDomainCached(address) + if address_resolved: + address = address_resolved + + site = self.get(address) + if not site: # Site not exist yet + site = self.add(address, *args, **kwargs) + return site + + def delete(self, address): + self.sites_changed = int(time.time()) + self.log.debug("Deleted site: %s" % address) + del(self.sites[address]) + # Delete from sites.json + self.save() + + # Lazy load sites + def list(self): + if not self.loaded: # Not loaded yet + self.log.debug("Sites not loaded yet...") + self.load(startup=True) + return self.sites + + +site_manager = SiteManager() # Singletone + +if config.action == "main": # Don't connect / add myself to peerlist + peer_blacklist = [("127.0.0.1", config.fileserver_port), ("::1", config.fileserver_port)] +else: + peer_blacklist = [] + diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py new file mode 100644 index 00000000..27032e79 --- /dev/null +++ b/src/Site/SiteStorage.py @@ -0,0 +1,636 @@ +import os +import re +import shutil +import json +import time +import errno +from collections import defaultdict + +import sqlite3 +import gevent.event + +import util +from util import SafeRe +from Db.Db import Db +from Debug import Debug +from Config import config +from util import helper +from util import ThreadPool +from Plugin import PluginManager +from Translate import translate as _ + + +thread_pool_fs_read = ThreadPool.ThreadPool(config.threads_fs_read, name="FS read") +thread_pool_fs_write = ThreadPool.ThreadPool(config.threads_fs_write, name="FS write") +thread_pool_fs_batch = ThreadPool.ThreadPool(1, name="FS batch") + + +@PluginManager.acceptPlugins +class SiteStorage(object): + def __init__(self, site, allow_create=True): + self.site = site + self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory + self.allowed_dir = os.path.abspath(self.directory) # Only serve file within this dir + self.log = site.log + self.db = None # Db class + self.db_checked = False # Checked db tables since startup + self.event_db_busy = None # Gevent AsyncResult if db is working on rebuild + self.has_db = self.isFile("dbschema.json") # The site has schema + + if not os.path.isdir(self.directory): + if allow_create: + os.mkdir(self.directory) # Create directory if not found + else: + raise Exception("Directory not exists: %s" % self.directory) + + def getDbFile(self): + if self.db: + return self.db.schema["db_file"] + else: + if self.isFile("dbschema.json"): + schema = self.loadJson("dbschema.json") + return schema["db_file"] + else: + return False + + # Create new databaseobject with the site's schema + def openDb(self, close_idle=False): + schema = self.getDbSchema() + db_path = self.getPath(schema["db_file"]) + return Db(schema, db_path, close_idle=close_idle) + + def closeDb(self, reason="Unknown (SiteStorage)"): + if self.db: + self.db.close(reason) + self.event_db_busy = None + self.db = None + + def getDbSchema(self): + try: + self.site.needFile("dbschema.json") + schema = self.loadJson("dbschema.json") + except Exception as err: + raise Exception("dbschema.json is not a valid JSON: %s" % err) + return schema + + def loadDb(self): + self.log.debug("No database, waiting for dbschema.json...") + self.site.needFile("dbschema.json", priority=3) + self.log.debug("Got dbschema.json") + self.has_db = self.isFile("dbschema.json") # Recheck if dbschema exist + if self.has_db: + schema = self.getDbSchema() + db_path = self.getPath(schema["db_file"]) + if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: + try: + self.rebuildDb(reason="Missing database") + except Exception as err: + self.log.error(err) + pass + + if self.db: + self.db.close("Gettig new db for SiteStorage") + self.db = self.openDb(close_idle=True) + try: + changed_tables = self.db.checkTables() + if changed_tables: + self.rebuildDb(delete_db=False, reason="Changed tables") # TODO: only update the changed table datas + except sqlite3.OperationalError: + pass + + # Return db class + @util.Noparallel() + def getDb(self): + if self.event_db_busy: # Db not ready for queries + self.log.debug("Wating for db...") + self.event_db_busy.get() # Wait for event + if not self.db: + self.loadDb() + return self.db + + def updateDbFile(self, inner_path, file=None, cur=None): + path = self.getPath(inner_path) + if cur: + db = cur.db + else: + db = self.getDb() + return db.updateJson(path, file, cur) + + # Return possible db files for the site + @thread_pool_fs_read.wrap + def getDbFiles(self): + found = 0 + for content_inner_path, content in self.site.content_manager.contents.items(): + # content.json file itself + if self.isFile(content_inner_path): + yield content_inner_path, self.getPath(content_inner_path) + else: + self.log.debug("[MISSING] %s" % content_inner_path) + # Data files in content.json + content_inner_path_dir = helper.getDirname(content_inner_path) # Content.json dir relative to site + for file_relative_path in list(content.get("files", {}).keys()) + list(content.get("files_optional", {}).keys()): + if not file_relative_path.endswith(".json") and not file_relative_path.endswith("json.gz"): + continue # We only interesed in json files + file_inner_path = content_inner_path_dir + file_relative_path # File Relative to site dir + file_inner_path = file_inner_path.strip("/") # Strip leading / + if self.isFile(file_inner_path): + yield file_inner_path, self.getPath(file_inner_path) + else: + self.log.debug("[MISSING] %s" % file_inner_path) + found += 1 + if found % 100 == 0: + time.sleep(0.001) # Context switch to avoid UI block + + # Rebuild sql cache + @util.Noparallel() + @thread_pool_fs_batch.wrap + def rebuildDb(self, delete_db=True, reason="Unknown"): + self.log.info("Rebuilding db (reason: %s)..." % reason) + self.has_db = self.isFile("dbschema.json") + if not self.has_db: + return False + + schema = self.loadJson("dbschema.json") + db_path = self.getPath(schema["db_file"]) + if os.path.isfile(db_path) and delete_db: + if self.db: + self.closeDb("rebuilding") # Close db if open + time.sleep(0.5) + self.log.info("Deleting %s" % db_path) + try: + os.unlink(db_path) + except Exception as err: + self.log.error("Delete error: %s" % err) + + if not self.db: + self.db = self.openDb() + self.event_db_busy = gevent.event.AsyncResult() + + self.log.info("Rebuild: Creating tables...") + + # raise DbTableError if not valid + self.db.checkTables() + + cur = self.db.getCursor() + cur.logging = False + s = time.time() + self.log.info("Rebuild: Getting db files...") + db_files = list(self.getDbFiles()) + num_imported = 0 + num_total = len(db_files) + num_error = 0 + + self.log.info("Rebuild: Importing data...") + try: + if num_total > 100: + self.site.messageWebsocket( + _["Database rebuilding...
Imported {0} of {1} files (error: {2})..."].format( + "0000", num_total, num_error + ), "rebuild", 0 + ) + for file_inner_path, file_path in db_files: + try: + if self.updateDbFile(file_inner_path, file=open(file_path, "rb"), cur=cur): + num_imported += 1 + except Exception as err: + self.log.error("Error importing %s: %s" % (file_inner_path, Debug.formatException(err))) + num_error += 1 + + if num_imported and num_imported % 100 == 0: + self.site.messageWebsocket( + _["Database rebuilding...
Imported {0} of {1} files (error: {2})..."].format( + num_imported, num_total, num_error + ), + "rebuild", int(float(num_imported) / num_total * 100) + ) + time.sleep(0.001) # Context switch to avoid UI block + + finally: + cur.close() + if num_total > 100: + self.site.messageWebsocket( + _["Database rebuilding...
Imported {0} of {1} files (error: {2})..."].format( + num_imported, num_total, num_error + ), "rebuild", 100 + ) + self.log.info("Rebuild: Imported %s data file in %.3fs" % (num_imported, time.time() - s)) + self.event_db_busy.set(True) # Event done, notify waiters + self.event_db_busy = None # Clear event + self.db.commit("Rebuilt") + + return True + + # Execute sql query or rebuild on dberror + def query(self, query, params=None): + if not query.strip().upper().startswith("SELECT"): + raise Exception("Only SELECT query supported") + + try: + res = self.getDb().execute(query, params) + except sqlite3.DatabaseError as err: + if err.__class__.__name__ == "DatabaseError": + self.log.error("Database error: %s, query: %s, try to rebuilding it..." % (err, query)) + try: + self.rebuildDb(reason="Query error") + except sqlite3.OperationalError: + pass + res = self.db.cur.execute(query, params) + else: + raise err + return res + + def ensureDir(self, inner_path): + try: + os.makedirs(self.getPath(inner_path)) + except OSError as err: + if err.errno == errno.EEXIST: + return False + else: + raise err + return True + + # Open file object + def open(self, inner_path, mode="rb", create_dirs=False, **kwargs): + file_path = self.getPath(inner_path) + if create_dirs: + file_inner_dir = os.path.dirname(inner_path) + self.ensureDir(file_inner_dir) + return open(file_path, mode, **kwargs) + + # Open file object + @thread_pool_fs_read.wrap + def read(self, inner_path, mode="rb"): + return self.open(inner_path, mode).read() + + @thread_pool_fs_write.wrap + def writeThread(self, inner_path, content): + file_path = self.getPath(inner_path) + # Create dir if not exist + self.ensureDir(os.path.dirname(inner_path)) + # Write file + if hasattr(content, 'read'): # File-like object + + with open(file_path, "wb") as file: + shutil.copyfileobj(content, file) # Write buff to disk + else: # Simple string + if inner_path == "content.json" and os.path.isfile(file_path): + helper.atomicWrite(file_path, content) + else: + with open(file_path, "wb") as file: + file.write(content) + + # Write content to file + def write(self, inner_path, content): + self.writeThread(inner_path, content) + self.onUpdated(inner_path) + + # Remove file from filesystem + def delete(self, inner_path): + file_path = self.getPath(inner_path) + os.unlink(file_path) + self.onUpdated(inner_path, file=False) + + def deleteDir(self, inner_path): + dir_path = self.getPath(inner_path) + os.rmdir(dir_path) + + def rename(self, inner_path_before, inner_path_after): + for retry in range(3): + rename_err = None + # To workaround "The process cannot access the file beacause it is being used by another process." error + try: + os.rename(self.getPath(inner_path_before), self.getPath(inner_path_after)) + break + except Exception as err: + rename_err = err + self.log.error("%s rename error: %s (retry #%s)" % (inner_path_before, err, retry)) + time.sleep(0.1 + retry) + if rename_err: + raise rename_err + + # List files from a directory + @thread_pool_fs_read.wrap + def walk(self, dir_inner_path, ignore=None): + directory = self.getPath(dir_inner_path) + for root, dirs, files in os.walk(directory): + root = root.replace("\\", "/") + root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/") + for file_name in files: + if root_relative_path: # Not root dir + file_relative_path = root_relative_path + "/" + file_name + else: + file_relative_path = file_name + + if ignore and SafeRe.match(ignore, file_relative_path): + continue + + yield file_relative_path + + # Don't scan directory that is in the ignore pattern + if ignore: + dirs_filtered = [] + for dir_name in dirs: + if root_relative_path: + dir_relative_path = root_relative_path + "/" + dir_name + else: + dir_relative_path = dir_name + + if ignore == ".*" or re.match(".*([|(]|^)%s([|)]|$)" % re.escape(dir_relative_path + "/.*"), ignore): + continue + + dirs_filtered.append(dir_name) + dirs[:] = dirs_filtered + + # list directories in a directory + @thread_pool_fs_read.wrap + def list(self, dir_inner_path): + directory = self.getPath(dir_inner_path) + return os.listdir(directory) + + # Site content updated + def onUpdated(self, inner_path, file=None): + # Update Sql cache + should_load_to_db = inner_path.endswith(".json") or inner_path.endswith(".json.gz") + if inner_path == "dbschema.json": + self.has_db = self.isFile("dbschema.json") + # Reopen DB to check changes + if self.has_db: + self.closeDb("New dbschema") + gevent.spawn(self.getDb) + elif not config.disable_db and should_load_to_db and self.has_db: # Load json file to db + if config.verbose: + self.log.debug("Loading json file to db: %s (file: %s)" % (inner_path, file)) + try: + self.updateDbFile(inner_path, file) + except Exception as err: + self.log.error("Json %s load error: %s" % (inner_path, Debug.formatException(err))) + self.closeDb("Json load error") + + # Load and parse json file + @thread_pool_fs_read.wrap + def loadJson(self, inner_path): + try: + with self.open(inner_path, "r", encoding="utf8") as file: + return json.load(file) + except Exception as err: + self.log.warning("Json load error: %s" % Debug.formatException(err)) + return None + + # Write formatted json file + def writeJson(self, inner_path, data): + # Write to disk + self.write(inner_path, helper.jsonDumps(data).encode("utf8")) + + # Get file size + def getSize(self, inner_path): + path = self.getPath(inner_path) + try: + return os.path.getsize(path) + except Exception: + return 0 + + # File exist + def isFile(self, inner_path): + return os.path.isfile(self.getPath(inner_path)) + + # File or directory exist + def isExists(self, inner_path): + return os.path.exists(self.getPath(inner_path)) + + # Dir exist + def isDir(self, inner_path): + return os.path.isdir(self.getPath(inner_path)) + + # Security check and return path of site's file + def getPath(self, inner_path): + inner_path = inner_path.replace("\\", "/") # Windows separator fix + if not inner_path: + return self.directory + + if "../" in inner_path: + raise Exception("File not allowed: %s" % inner_path) + + return "%s/%s" % (self.directory, inner_path) + + # Get site dir relative path + def getInnerPath(self, path): + if path == self.directory: + inner_path = "" + else: + if path.startswith(self.directory): + inner_path = path[len(self.directory) + 1:] + else: + raise Exception("File not allowed: %s" % path) + return inner_path + + # Verify all files sha512sum using content.json + def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True): + bad_files = [] + back = defaultdict(int) + back["bad_files"] = bad_files + i = 0 + self.log.debug("Verifing files...") + + if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first + self.log.debug("VerifyFile content.json not exists") + self.site.needFile("content.json", update=True) # Force update to fix corrupt file + self.site.content_manager.loadContent() # Reload content.json + for content_inner_path, content in list(self.site.content_manager.contents.items()): + back["num_content"] += 1 + i += 1 + if i % 50 == 0: + time.sleep(0.001) # Context switch to avoid gevent hangs + if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file + back["num_content_missing"] += 1 + self.log.debug("[MISSING] %s" % content_inner_path) + bad_files.append(content_inner_path) + + for file_relative_path in list(content.get("files", {}).keys()): + back["num_file"] += 1 + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir + file_inner_path = file_inner_path.strip("/") # Strip leading / + file_path = self.getPath(file_inner_path) + if not os.path.isfile(file_path): + back["num_file_missing"] += 1 + self.log.debug("[MISSING] %s" % file_inner_path) + bad_files.append(file_inner_path) + continue + + if quick_check: + ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"] + if not ok: + err = "Invalid size" + else: + try: + ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb")) + except Exception as _err: + err = _err + ok = False + + if not ok: + back["num_file_invalid"] += 1 + self.log.debug("[INVALID] %s: %s" % (file_inner_path, err)) + if add_changed or content.get("cert_user_id"): # If updating own site only add changed user files + bad_files.append(file_inner_path) + + # Optional files + optional_added = 0 + optional_removed = 0 + for file_relative_path in list(content.get("files_optional", {}).keys()): + back["num_optional"] += 1 + file_node = content["files_optional"][file_relative_path] + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir + file_inner_path = file_inner_path.strip("/") # Strip leading / + file_path = self.getPath(file_inner_path) + hash_id = self.site.content_manager.hashfield.getHashId(file_node["sha512"]) + if not os.path.isfile(file_path): + if self.site.content_manager.isDownloaded(file_inner_path, hash_id): + back["num_optional_removed"] += 1 + self.log.debug("[OPTIONAL MISSING] %s" % file_inner_path) + self.site.content_manager.optionalRemoved(file_inner_path, hash_id, file_node["size"]) + if add_optional and self.site.isDownloadable(file_inner_path): + self.log.debug("[OPTIONAL ADDING] %s" % file_inner_path) + bad_files.append(file_inner_path) + continue + + if quick_check: + ok = os.path.getsize(file_path) == content["files_optional"][file_relative_path]["size"] + else: + try: + ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb")) + except Exception as err: + ok = False + + if ok: + if not self.site.content_manager.isDownloaded(file_inner_path, hash_id): + back["num_optional_added"] += 1 + self.site.content_manager.optionalDownloaded(file_inner_path, hash_id, file_node["size"]) + optional_added += 1 + self.log.debug("[OPTIONAL FOUND] %s" % file_inner_path) + else: + if self.site.content_manager.isDownloaded(file_inner_path, hash_id): + back["num_optional_removed"] += 1 + self.site.content_manager.optionalRemoved(file_inner_path, hash_id, file_node["size"]) + optional_removed += 1 + bad_files.append(file_inner_path) + self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path) + + if config.verbose: + self.log.debug( + "%s verified: %s, quick: %s, optionals: +%s -%s" % + (content_inner_path, len(content["files"]), quick_check, optional_added, optional_removed) + ) + + self.site.content_manager.contents.db.processDelayed() + time.sleep(0.001) # Context switch to avoid gevent hangs + return back + + # Check and try to fix site files integrity + def updateBadFiles(self, quick_check=True): + s = time.time() + res = self.verifyFiles( + quick_check, + add_optional=True, + add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned + ) + bad_files = res["bad_files"] + self.site.bad_files = {} + if bad_files: + for bad_file in bad_files: + self.site.bad_files[bad_file] = 1 + self.log.debug("Checked files in %.2fs... Found bad files: %s, Quick:%s" % (time.time() - s, len(bad_files), quick_check)) + + # Delete site's all file + @thread_pool_fs_batch.wrap + def deleteFiles(self): + site_title = self.site.content_manager.contents.get("content.json", {}).get("title", self.site.address) + message_id = "delete-%s" % self.site.address + self.log.debug("Deleting files from content.json (title: %s)..." % site_title) + + files = [] # Get filenames + content_inner_paths = list(self.site.content_manager.contents.keys()) + for i, content_inner_path in enumerate(content_inner_paths): + content = self.site.content_manager.contents.get(content_inner_path, {}) + files.append(content_inner_path) + # Add normal files + for file_relative_path in list(content.get("files", {}).keys()): + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir + files.append(file_inner_path) + # Add optional files + for file_relative_path in list(content.get("files_optional", {}).keys()): + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir + files.append(file_inner_path) + + if i % 100 == 0: + num_files = len(files) + self.site.messageWebsocket( + _("Deleting site {site_title}...
Collected {num_files} files"), + message_id, (i / len(content_inner_paths)) * 25 + ) + + if self.isFile("dbschema.json"): + self.log.debug("Deleting db file...") + self.closeDb("Deleting site") + self.has_db = False + try: + schema = self.loadJson("dbschema.json") + db_path = self.getPath(schema["db_file"]) + if os.path.isfile(db_path): + os.unlink(db_path) + except Exception as err: + self.log.error("Db file delete error: %s" % err) + + num_files = len(files) + for i, inner_path in enumerate(files): + path = self.getPath(inner_path) + if os.path.isfile(path): + for retry in range(5): + try: + os.unlink(path) + break + except Exception as err: + self.log.error("Error removing %s: %s, try #%s" % (inner_path, err, retry)) + time.sleep(float(retry) / 10) + if i % 100 == 0: + self.site.messageWebsocket( + _("Deleting site {site_title}...
Deleting file {i}/{num_files}"), + message_id, 25 + (i / num_files) * 50 + ) + self.onUpdated(inner_path, False) + + self.log.debug("Deleting empty dirs...") + i = 0 + for root, dirs, files in os.walk(self.directory, topdown=False): + for dir in dirs: + path = os.path.join(root, dir) + if os.path.isdir(path): + try: + i += 1 + if i % 100 == 0: + self.site.messageWebsocket( + _("Deleting site {site_title}...
Deleting empty directories {i}"), + message_id, 85 + ) + os.rmdir(path) + except OSError: # Not empty + pass + + if os.path.isdir(self.directory) and os.listdir(self.directory) == []: + os.rmdir(self.directory) # Remove sites directory if empty + + if os.path.isdir(self.directory): + self.log.debug("Some unknown file remained in site data dir: %s..." % self.directory) + self.site.messageWebsocket( + _("Deleting site {site_title}...
Site deleted, but some unknown files left in the directory"), + message_id, 100 + ) + return False # Some files not deleted + else: + self.log.debug("Site %s data directory deleted: %s..." % (site_title, self.directory)) + + self.site.messageWebsocket( + _("Deleting site {site_title}...
All files deleted successfully"), + message_id, 100 + ) + + return True # All clean diff --git a/src/Site/__init__.py b/src/Site/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Test/BenchmarkSsl.py b/src/Test/BenchmarkSsl.py new file mode 100644 index 00000000..06181b89 --- /dev/null +++ b/src/Test/BenchmarkSsl.py @@ -0,0 +1,162 @@ +#!/usr/bin/python2 +from gevent import monkey +monkey.patch_all() +import os +import time +import sys +import socket +import ssl +sys.path.append(os.path.abspath("..")) # Imports relative to src dir + +import io as StringIO +import gevent + +from gevent.server import StreamServer +from gevent.pool import Pool +from Config import config +config.parse() +from util import SslPatch + +# Server +socks = [] +data = os.urandom(1024 * 100) +data += "\n" + + +def handle(sock_raw, addr): + socks.append(sock_raw) + sock = sock_raw + # sock = ctx.wrap_socket(sock, server_side=True) + # if sock_raw.recv( 1, gevent.socket.MSG_PEEK ) == "\x16": + # sock = gevent.ssl.wrap_socket(sock_raw, server_side=True, keyfile='key-cz.pem', + # certfile='cert-cz.pem', ciphers=ciphers, ssl_version=ssl.PROTOCOL_TLSv1) + # fp = os.fdopen(sock.fileno(), 'rb', 1024*512) + try: + while True: + line = sock.recv(16 * 1024) + if not line: + break + if line == "bye\n": + break + elif line == "gotssl\n": + sock.sendall("yes\n") + sock = gevent.ssl.wrap_socket( + sock_raw, server_side=True, keyfile='../../data/key-rsa.pem', certfile='../../data/cert-rsa.pem', + ciphers=ciphers, ssl_version=ssl.PROTOCOL_TLSv1 + ) + else: + sock.sendall(data) + except Exception as err: + print(err) + try: + sock.shutdown(gevent.socket.SHUT_WR) + sock.close() + except: + pass + socks.remove(sock_raw) + +pool = Pool(1000) # do not accept more than 10000 connections +server = StreamServer(('127.0.0.1', 1234), handle) +server.start() + + +# Client + + +total_num = 0 +total_bytes = 0 +clipher = None +ciphers = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDH+AES128:ECDHE-RSA-AES128-GCM-SHA256:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:HIGH:" + \ + "!aNULL:!eNULL:!EXPORT:!DSS:!DES:!RC4:!3DES:!MD5:!PSK" + +# ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + + +def getData(): + global total_num, total_bytes, clipher + data = None + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # sock = socket.ssl(s) + # sock = ssl.wrap_socket(sock) + sock.connect(("127.0.0.1", 1234)) + # sock.do_handshake() + # clipher = sock.cipher() + sock.send("gotssl\n") + if sock.recv(128) == "yes\n": + sock = ssl.wrap_socket(sock, ciphers=ciphers, ssl_version=ssl.PROTOCOL_TLSv1) + sock.do_handshake() + clipher = sock.cipher() + + for req in range(20): + sock.sendall("req\n") + buff = StringIO.StringIO() + data = sock.recv(16 * 1024) + buff.write(data) + if not data: + break + while not data.endswith("\n"): + data = sock.recv(16 * 1024) + if not data: + break + buff.write(data) + total_num += 1 + total_bytes += buff.tell() + if not data: + print("No data") + + sock.shutdown(gevent.socket.SHUT_WR) + sock.close() + +s = time.time() + + +def info(): + import psutil + import os + process = psutil.Process(os.getpid()) + if "memory_info" in dir(process): + memory_info = process.memory_info + else: + memory_info = process.get_memory_info + while 1: + print(total_num, "req", (total_bytes / 1024), "kbytes", "transfered in", time.time() - s, end=' ') + print("using", clipher, "Mem:", memory_info()[0] / float(2 ** 20)) + time.sleep(1) + +gevent.spawn(info) + +for test in range(1): + clients = [] + for i in range(500): # Thread + clients.append(gevent.spawn(getData)) + gevent.joinall(clients) + + +print(total_num, "req", (total_bytes / 1024), "kbytes", "transfered in", time.time() - s) + +# Separate client/server process: +# 10*10*100: +# Raw: 10000 req 1000009 kbytes transfered in 5.39999985695 +# RSA 2048: 10000 req 1000009 kbytes transfered in 27.7890000343 using ('ECDHE-RSA-AES256-SHA', 'TLSv1/SSLv3', 256) +# ECC: 10000 req 1000009 kbytes transfered in 26.1959998608 using ('ECDHE-ECDSA-AES256-SHA', 'TLSv1/SSLv3', 256) +# ECC: 10000 req 1000009 kbytes transfered in 28.2410001755 using ('ECDHE-ECDSA-AES256-GCM-SHA384', 'TLSv1/SSLv3', 256) Mem: 13.3828125 +# +# 10*100*10: +# Raw: 10000 req 1000009 kbytes transfered in 7.02700018883 Mem: 14.328125 +# RSA 2048: 10000 req 1000009 kbytes transfered in 44.8860001564 using ('ECDHE-RSA-AES256-GCM-SHA384', 'TLSv1/SSLv3', 256) Mem: 20.078125 +# ECC: 10000 req 1000009 kbytes transfered in 37.9430000782 using ('ECDHE-ECDSA-AES256-GCM-SHA384', 'TLSv1/SSLv3', 256) Mem: 20.0234375 +# +# 1*100*100: +# Raw: 10000 req 1000009 kbytes transfered in 4.64400005341 Mem: 14.06640625 +# RSA: 10000 req 1000009 kbytes transfered in 24.2300000191 using ('ECDHE-RSA-AES256-GCM-SHA384', 'TLSv1/SSLv3', 256) Mem: 19.7734375 +# ECC: 10000 req 1000009 kbytes transfered in 22.8849999905 using ('ECDHE-ECDSA-AES256-GCM-SHA384', 'TLSv1/SSLv3', 256) Mem: 17.8125 +# AES128: 10000 req 1000009 kbytes transfered in 21.2839999199 using ('AES128-GCM-SHA256', 'TLSv1/SSLv3', 128) Mem: 14.1328125 +# ECC+128: 10000 req 1000009 kbytes transfered in 20.496999979 using ('ECDHE-ECDSA-AES128-GCM-SHA256', 'TLSv1/SSLv3', 128) Mem: 14.40234375 +# +# +# Single process: +# 1*100*100 +# RSA: 10000 req 1000009 kbytes transfered in 41.7899999619 using ('ECDHE-RSA-AES128-GCM-SHA256', 'TLSv1/SSLv3', 128) Mem: 26.91015625 +# +# 10*10*100 +# RSA: 10000 req 1000009 kbytes transfered in 40.1640000343 using ('ECDHE-RSA-AES128-GCM-SHA256', 'TLSv1/SSLv3', 128) Mem: 14.94921875 diff --git a/src/Test/Spy.py b/src/Test/Spy.py new file mode 100644 index 00000000..44422550 --- /dev/null +++ b/src/Test/Spy.py @@ -0,0 +1,23 @@ +import logging + +class Spy: + def __init__(self, obj, func_name): + self.obj = obj + self.__name__ = func_name + self.func_original = getattr(self.obj, func_name) + self.calls = [] + + def __enter__(self, *args, **kwargs): + logging.debug("Spy started") + def loggedFunc(cls, *args, **kwargs): + call = dict(enumerate(args, 1)) + call[0] = cls + call.update(kwargs) + logging.debug("Spy call: %s" % call) + self.calls.append(call) + return self.func_original(cls, *args, **kwargs) + setattr(self.obj, self.__name__, loggedFunc) + return self.calls + + def __exit__(self, *args, **kwargs): + setattr(self.obj, self.__name__, self.func_original) \ No newline at end of file diff --git a/src/Test/TestCached.py b/src/Test/TestCached.py new file mode 100644 index 00000000..088962c0 --- /dev/null +++ b/src/Test/TestCached.py @@ -0,0 +1,59 @@ +import time + +from util import Cached + + +class CachedObject: + def __init__(self): + self.num_called_add = 0 + self.num_called_multiply = 0 + self.num_called_none = 0 + + @Cached(timeout=1) + def calcAdd(self, a, b): + self.num_called_add += 1 + return a + b + + @Cached(timeout=1) + def calcMultiply(self, a, b): + self.num_called_multiply += 1 + return a * b + + @Cached(timeout=1) + def none(self): + self.num_called_none += 1 + return None + + +class TestCached: + def testNoneValue(self): + cached_object = CachedObject() + assert cached_object.none() is None + assert cached_object.none() is None + assert cached_object.num_called_none == 1 + time.sleep(2) + assert cached_object.none() is None + assert cached_object.num_called_none == 2 + + def testCall(self): + cached_object = CachedObject() + + assert cached_object.calcAdd(1, 2) == 3 + assert cached_object.calcAdd(1, 2) == 3 + assert cached_object.calcMultiply(1, 2) == 2 + assert cached_object.calcMultiply(1, 2) == 2 + assert cached_object.num_called_add == 1 + assert cached_object.num_called_multiply == 1 + + assert cached_object.calcAdd(2, 3) == 5 + assert cached_object.calcAdd(2, 3) == 5 + assert cached_object.num_called_add == 2 + + assert cached_object.calcAdd(1, 2) == 3 + assert cached_object.calcMultiply(2, 3) == 6 + assert cached_object.num_called_add == 2 + assert cached_object.num_called_multiply == 2 + + time.sleep(2) + assert cached_object.calcAdd(1, 2) == 3 + assert cached_object.num_called_add == 3 diff --git a/src/Test/TestConfig.py b/src/Test/TestConfig.py new file mode 100644 index 00000000..24084392 --- /dev/null +++ b/src/Test/TestConfig.py @@ -0,0 +1,31 @@ +import pytest + +import Config + + +@pytest.mark.usefixtures("resetSettings") +class TestConfig: + def testParse(self): + # Defaults + config_test = Config.Config("zeronet.py".split(" ")) + config_test.parse(silent=True, parse_config=False) + assert not config_test.debug + assert not config_test.debug_socket + + # Test parse command line with unknown parameters (ui_password) + config_test = Config.Config("zeronet.py --debug --debug_socket --ui_password hello".split(" ")) + config_test.parse(silent=True, parse_config=False) + assert config_test.debug + assert config_test.debug_socket + with pytest.raises(AttributeError): + config_test.ui_password + + # More complex test + args = "zeronet.py --unknown_arg --debug --debug_socket --ui_restrict 127.0.0.1 1.2.3.4 " + args += "--another_unknown argument --use_openssl False siteSign address privatekey --inner_path users/content.json" + config_test = Config.Config(args.split(" ")) + config_test.parse(silent=True, parse_config=False) + assert config_test.debug + assert "1.2.3.4" in config_test.ui_restrict + assert not config_test.use_openssl + assert config_test.inner_path == "users/content.json" diff --git a/src/Test/TestConnectionServer.py b/src/Test/TestConnectionServer.py new file mode 100644 index 00000000..82ee605c --- /dev/null +++ b/src/Test/TestConnectionServer.py @@ -0,0 +1,118 @@ +import time +import socket +import gevent + +import pytest +import mock + +from Crypt import CryptConnection +from Connection import ConnectionServer +from Config import config + + +@pytest.mark.usefixtures("resetSettings") +class TestConnection: + def testIpv6(self, file_server6): + assert ":" in file_server6.ip + + client = ConnectionServer(file_server6.ip, 1545) + connection = client.getConnection(file_server6.ip, 1544) + + assert connection.ping() + + # Close connection + connection.close() + client.stop() + time.sleep(0.01) + assert len(file_server6.connections) == 0 + + # Should not able to reach on ipv4 ip + with pytest.raises(socket.error) as err: + client = ConnectionServer("127.0.0.1", 1545) + connection = client.getConnection("127.0.0.1", 1544) + + def testSslConnection(self, file_server): + client = ConnectionServer(file_server.ip, 1545) + assert file_server != client + + # Connect to myself + with mock.patch('Config.config.ip_local', return_value=[]): # SSL not used for local ips + connection = client.getConnection(file_server.ip, 1544) + + assert len(file_server.connections) == 1 + assert connection.handshake + assert connection.crypt + + + # Close connection + connection.close("Test ended") + client.stop() + time.sleep(0.1) + assert len(file_server.connections) == 0 + assert file_server.num_incoming == 2 # One for file_server fixture, one for the test + + def testRawConnection(self, file_server): + client = ConnectionServer(file_server.ip, 1545) + assert file_server != client + + # Remove all supported crypto + crypt_supported_bk = CryptConnection.manager.crypt_supported + CryptConnection.manager.crypt_supported = [] + + with mock.patch('Config.config.ip_local', return_value=[]): # SSL not used for local ips + connection = client.getConnection(file_server.ip, 1544) + assert len(file_server.connections) == 1 + assert not connection.crypt + + # Close connection + connection.close() + client.stop() + time.sleep(0.01) + assert len(file_server.connections) == 0 + + # Reset supported crypts + CryptConnection.manager.crypt_supported = crypt_supported_bk + + def testPing(self, file_server, site): + client = ConnectionServer(file_server.ip, 1545) + connection = client.getConnection(file_server.ip, 1544) + + assert connection.ping() + + connection.close() + client.stop() + + def testGetConnection(self, file_server): + client = ConnectionServer(file_server.ip, 1545) + connection = client.getConnection(file_server.ip, 1544) + + # Get connection by ip/port + connection2 = client.getConnection(file_server.ip, 1544) + assert connection == connection2 + + # Get connection by peerid + assert not client.getConnection(file_server.ip, 1544, peer_id="notexists", create=False) + connection2 = client.getConnection(file_server.ip, 1544, peer_id=connection.handshake["peer_id"], create=False) + assert connection2 == connection + + connection.close() + client.stop() + + def testFloodProtection(self, file_server): + whitelist = file_server.whitelist # Save for reset + file_server.whitelist = [] # Disable 127.0.0.1 whitelist + client = ConnectionServer(file_server.ip, 1545) + + # Only allow 6 connection in 1 minute + for reconnect in range(6): + connection = client.getConnection(file_server.ip, 1544) + assert connection.handshake + connection.close() + + # The 7. one will timeout + with pytest.raises(gevent.Timeout): + with gevent.Timeout(0.1): + connection = client.getConnection(file_server.ip, 1544) + + # Reset whitelist + file_server.whitelist = whitelist diff --git a/src/Test/TestContent.py b/src/Test/TestContent.py new file mode 100644 index 00000000..7e7ca1a5 --- /dev/null +++ b/src/Test/TestContent.py @@ -0,0 +1,273 @@ +import json +import time +import io + +import pytest + +from Crypt import CryptBitcoin +from Content.ContentManager import VerifyError, SignError +from util.SafeRe import UnsafePatternError + + +@pytest.mark.usefixtures("resetSettings") +class TestContent: + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" + + def testInclude(self, site): + # Rules defined in parent content.json + rules = site.content_manager.getRules("data/test_include/content.json") + + assert rules["signers"] == ["15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo"] # Valid signer + assert rules["user_name"] == "test" # Extra data + assert rules["max_size"] == 20000 # Max size of files + assert not rules["includes_allowed"] # Don't allow more includes + assert rules["files_allowed"] == "data.json" # Allowed file pattern + + # Valid signers for "data/test_include/content.json" + valid_signers = site.content_manager.getValidSigners("data/test_include/content.json") + assert "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" in valid_signers # Extra valid signer defined in parent content.json + assert "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" in valid_signers # The site itself + assert len(valid_signers) == 2 # No more + + # Valid signers for "data/users/content.json" + valid_signers = site.content_manager.getValidSigners("data/users/content.json") + assert "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" in valid_signers # Extra valid signer defined in parent content.json + assert "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" in valid_signers # The site itself + assert len(valid_signers) == 2 + + # Valid signers for root content.json + assert site.content_manager.getValidSigners("content.json") == ["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] + + def testInlcudeLimits(self, site, crypt_bitcoin_lib): + # Data validation + res = [] + data_dict = { + "files": { + "data.json": { + "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", + "size": 505 + } + }, + "modified": time.time() + } + + # Normal data + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey)} + data_json = json.dumps(data_dict).encode() + data = io.BytesIO(data_json) + assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + + # Reset + del data_dict["signs"] + + # Too large + data_dict["files"]["data.json"]["size"] = 200000 # Emulate 2MB sized data.json + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey)} + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + assert "Include too large" in str(err.value) + + # Reset + data_dict["files"]["data.json"]["size"] = 505 + del data_dict["signs"] + + # Not allowed file + data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"] + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey)} + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + assert "File not allowed" in str(err.value) + + # Reset + del data_dict["files"]["notallowed.exe"] + del data_dict["signs"] + + # Should work again + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey)} + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + + @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"]) + def testSign(self, site, inner_path): + # Bad privatekey + with pytest.raises(SignError) as err: + site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False) + assert "Private key invalid" in str(err.value) + + # Good privatekey + content = site.content_manager.sign(inner_path, privatekey=self.privatekey, filewrite=False) + content_old = site.content_manager.contents[inner_path] # Content before the sign + assert not content_old == content # Timestamp changed + assert site.address in content["signs"] # Used the site's private key to sign + if inner_path == "content.json": + assert len(content["files"]) == 17 + elif inner_path == "data/test-include/content.json": + assert len(content["files"]) == 1 + elif inner_path == "data/users/content.json": + assert len(content["files"]) == 0 + + # Everything should be same as before except the modified timestamp and the signs + assert ( + {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign", "zeronet_version"]} + == + {key: val for key, val in content.items() if key not in ["modified", "signs", "sign", "zeronet_version"]} + ) + + def testSignOptionalFiles(self, site): + for hash in list(site.content_manager.hashfield): + site.content_manager.hashfield.remove(hash) + + assert len(site.content_manager.hashfield) == 0 + + site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))" + content_optional = site.content_manager.sign(privatekey=self.privatekey, filewrite=False, remove_missing_optional=True) + + del site.content_manager.contents["content.json"]["optional"] + content_nooptional = site.content_manager.sign(privatekey=self.privatekey, filewrite=False, remove_missing_optional=True) + + assert len(content_nooptional.get("files_optional", {})) == 0 # No optional files if no pattern + assert len(content_optional["files_optional"]) > 0 + assert len(site.content_manager.hashfield) == len(content_optional["files_optional"]) # Hashed optional files should be added to hashfield + assert len(content_nooptional["files"]) > len(content_optional["files"]) + + def testFileInfo(self, site): + assert "sha512" in site.content_manager.getFileInfo("index.html") + assert site.content_manager.getFileInfo("data/img/domain.png")["content_inner_path"] == "content.json" + assert site.content_manager.getFileInfo("data/users/hello.png")["content_inner_path"] == "data/users/content.json" + assert site.content_manager.getFileInfo("data/users/content.json")["content_inner_path"] == "data/users/content.json" + assert not site.content_manager.getFileInfo("notexist") + + # Optional file + file_info_optional = site.content_manager.getFileInfo("data/optional.txt") + assert "sha512" in file_info_optional + assert file_info_optional["optional"] is True + + # Not exists yet user content.json + assert "cert_signers" in site.content_manager.getFileInfo("data/users/unknown/content.json") + + # Optional user file + file_info_optional = site.content_manager.getFileInfo("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert "sha512" in file_info_optional + assert file_info_optional["optional"] is True + + def testVerify(self, site, crypt_bitcoin_lib): + inner_path = "data/test_include/content.json" + data_dict = site.storage.loadJson(inner_path) + data = io.BytesIO(json.dumps(data_dict).encode("utf8")) + + # Re-sign + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) + } + assert site.content_manager.verifyFile(inner_path, data, ignore_same=False) + + # Wrong address + data_dict["address"] = "Othersite" + del data_dict["signs"] + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(inner_path, data, ignore_same=False) + assert "Wrong site address" in str(err.value) + + # Wrong inner_path + data_dict["address"] = "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" + data_dict["inner_path"] = "content.json" + del data_dict["signs"] + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(inner_path, data, ignore_same=False) + assert "Wrong inner_path" in str(err.value) + + # Everything right again + data_dict["address"] = "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" + data_dict["inner_path"] = inner_path + del data_dict["signs"] + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile(inner_path, data, ignore_same=False) + + def testVerifyInnerPath(self, site, crypt_bitcoin_lib): + inner_path = "content.json" + data_dict = site.storage.loadJson(inner_path) + + for good_relative_path in ["data.json", "out/data.json", "Any File [by none] (1).jpg", "ÃĄrvzítÅąrő/tÃŧkÃļrfÃērÃŗgÊp.txt"]: + data_dict["files"] = {good_relative_path: {"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505}} + + if "sign" in data_dict: + del data_dict["sign"] + del data_dict["signs"] + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile(inner_path, data, ignore_same=False) + + for bad_relative_path in ["../data.json", "data/" * 100, "invalid|file.jpg", "con.txt", "any/con.txt"]: + data_dict["files"] = {bad_relative_path: {"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505}} + + if "sign" in data_dict: + del data_dict["sign"] + del data_dict["signs"] + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(inner_path, data, ignore_same=False) + assert "Invalid relative path" in str(err.value) + + @pytest.mark.parametrize("key", ["ignore", "optional"]) + def testSignUnsafePattern(self, site, key): + site.content_manager.contents["content.json"][key] = "([a-zA-Z]+)*" + with pytest.raises(UnsafePatternError) as err: + site.content_manager.sign("content.json", privatekey=self.privatekey, filewrite=False) + assert "Potentially unsafe" in str(err.value) + + + def testVerifyUnsafePattern(self, site, crypt_bitcoin_lib): + site.content_manager.contents["content.json"]["includes"]["data/test_include/content.json"]["files_allowed"] = "([a-zA-Z]+)*" + with pytest.raises(UnsafePatternError) as err: + with site.storage.open("data/test_include/content.json") as data: + site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + assert "Potentially unsafe" in str(err.value) + + site.content_manager.contents["data/users/content.json"]["user_contents"]["permission_rules"]["([a-zA-Z]+)*"] = {"max_size": 0} + with pytest.raises(UnsafePatternError) as err: + with site.storage.open("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json") as data: + site.content_manager.verifyFile("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", data, ignore_same=False) + assert "Potentially unsafe" in str(err.value) + + def testPathValidation(self, site): + assert site.content_manager.isValidRelativePath("test.txt") + assert site.content_manager.isValidRelativePath("test/!@#$%^&().txt") + assert site.content_manager.isValidRelativePath("ÜøßÂŒƂÆÇ.txt") + assert site.content_manager.isValidRelativePath("Ņ‚ĐĩŅŅ‚.Ņ‚ĐĩĐēŅŅ‚") + assert site.content_manager.isValidRelativePath("𝐮𝐧đĸ𝐜𝐨𝐝𝐞𝑖𝑠𝒂𝒘𝒆𝒔𝒐𝒎𝒆") + + # Test rules based on https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names + + assert not site.content_manager.isValidRelativePath("any\\hello.txt") # \ not allowed + assert not site.content_manager.isValidRelativePath("/hello.txt") # Cannot start with / + assert not site.content_manager.isValidRelativePath("\\hello.txt") # Cannot start with \ + assert not site.content_manager.isValidRelativePath("../hello.txt") # Not allowed .. in path + assert not site.content_manager.isValidRelativePath("\0hello.txt") # NULL character + assert not site.content_manager.isValidRelativePath("\31hello.txt") # 0-31 (ASCII control characters) + assert not site.content_manager.isValidRelativePath("any/hello.txt ") # Cannot end with space + assert not site.content_manager.isValidRelativePath("any/hello.txt.") # Cannot end with dot + assert site.content_manager.isValidRelativePath(".hello.txt") # Allow start with dot + assert not site.content_manager.isValidRelativePath("any/CON") # Protected names on Windows + assert not site.content_manager.isValidRelativePath("CON/any.txt") + assert not site.content_manager.isValidRelativePath("any/lpt1.txt") + assert site.content_manager.isValidRelativePath("any/CONAN") + assert not site.content_manager.isValidRelativePath("any/CONOUT$") + assert not site.content_manager.isValidRelativePath("a" * 256) # Max 255 characters allowed diff --git a/src/Test/TestContentUser.py b/src/Test/TestContentUser.py new file mode 100644 index 00000000..8e91dd3e --- /dev/null +++ b/src/Test/TestContentUser.py @@ -0,0 +1,390 @@ +import json +import io + +import pytest + +from Crypt import CryptBitcoin +from Content.ContentManager import VerifyError, SignError + + +@pytest.mark.usefixtures("resetSettings") +class TestContentUser: + def testSigners(self, site): + # File info for not existing user file + file_info = site.content_manager.getFileInfo("data/users/notexist/data.json") + assert file_info["content_inner_path"] == "data/users/notexist/content.json" + file_info = site.content_manager.getFileInfo("data/users/notexist/a/b/data.json") + assert file_info["content_inner_path"] == "data/users/notexist/content.json" + valid_signers = site.content_manager.getValidSigners("data/users/notexist/content.json") + assert valid_signers == ["14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "notexist", "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] + + # File info for exsitsing user file + valid_signers = site.content_manager.getValidSigners("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json") + assert '1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT' in valid_signers # The site address + assert '14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet' in valid_signers # Admin user defined in data/users/content.json + assert '1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C' in valid_signers # The user itself + assert len(valid_signers) == 3 # No more valid signers + + # Valid signer for banned user + user_content = site.storage.loadJson("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json") + user_content["cert_user_id"] = "bad@zeroid.bit" + + valid_signers = site.content_manager.getValidSigners("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + assert '1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT' in valid_signers # The site address + assert '14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet' in valid_signers # Admin user defined in data/users/content.json + assert '1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C' not in valid_signers # The user itself + + def testRules(self, site): + # We going to manipulate it this test rules based on data/users/content.json + user_content = site.storage.loadJson("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json") + + # Known user + user_content["cert_auth_type"] = "web" + user_content["cert_user_id"] = "nofish@zeroid.bit" + rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + assert rules["max_size"] == 100000 + assert "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" in rules["signers"] + + # Unknown user + user_content["cert_auth_type"] = "web" + user_content["cert_user_id"] = "noone@zeroid.bit" + rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + assert rules["max_size"] == 10000 + assert "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" in rules["signers"] + + # User with more size limit based on auth type + user_content["cert_auth_type"] = "bitmsg" + user_content["cert_user_id"] = "noone@zeroid.bit" + rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + assert rules["max_size"] == 15000 + assert "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" in rules["signers"] + + # Banned user + user_content["cert_auth_type"] = "web" + user_content["cert_user_id"] = "bad@zeroid.bit" + rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + assert "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" not in rules["signers"] + + def testRulesAddress(self, site): + user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" + user_content = site.storage.loadJson(user_inner_path) + + rules = site.content_manager.getRules(user_inner_path, user_content) + assert rules["max_size"] == 10000 + assert "1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9" in rules["signers"] + + users_content = site.content_manager.contents["data/users/content.json"] + + # Ban user based on address + users_content["user_contents"]["permissions"]["1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9"] = False + rules = site.content_manager.getRules(user_inner_path, user_content) + assert "1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9" not in rules["signers"] + + # Change max allowed size + users_content["user_contents"]["permissions"]["1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9"] = {"max_size": 20000} + rules = site.content_manager.getRules(user_inner_path, user_content) + assert rules["max_size"] == 20000 + + def testVerifyAddress(self, site): + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT + user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" + data_dict = site.storage.loadJson(user_inner_path) + users_content = site.content_manager.contents["data/users/content.json"] + + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # Test error on 15k data.json + data_dict["files"]["data.json"]["size"] = 1024 * 15 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + assert "Include too large" in str(err.value) + + # Give more space based on address + users_content["user_contents"]["permissions"]["1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9"] = {"max_size": 20000} + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + def testVerify(self, site): + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT + user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" + data_dict = site.storage.loadJson(user_inner_path) + users_content = site.content_manager.contents["data/users/content.json"] + + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # Test max size exception by setting allowed to 0 + rules = site.content_manager.getRules(user_inner_path, data_dict) + assert rules["max_size"] == 10000 + assert users_content["user_contents"]["permission_rules"][".*"]["max_size"] == 10000 + + users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 0 + rules = site.content_manager.getRules(user_inner_path, data_dict) + assert rules["max_size"] == 0 + data = io.BytesIO(json.dumps(data_dict).encode()) + + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + assert "Include too large" in str(err.value) + users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 10000 # Reset + + # Test max optional size exception + # 1 MB gif = Allowed + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # 100 MB gif = Not allowed + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 100 * 1024 * 1024 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + assert "Include optional files too large" in str(err.value) + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 # Reset + + # hello.exe = Not allowed + data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"]["peanut-butter-jelly-time.gif"] + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + assert "Optional file not allowed" in str(err.value) + del data_dict["files_optional"]["hello.exe"] # Reset + + # Includes not allowed in user content + data_dict["includes"] = {"other.json": {}} + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = io.BytesIO(json.dumps(data_dict).encode()) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + assert "Includes not allowed" in str(err.value) + + def testCert(self, site): + # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" + user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" + # cert_addr = "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" + cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" + + # Check if the user file is loaded + assert "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" in site.content_manager.contents + user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] + rules_content = site.content_manager.contents["data/users/content.json"] + + # Override valid cert signers for the test + rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [ + "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", + "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" + ] + + # Check valid cert signers + rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + assert rules["cert_signers"] == {"zeroid.bit": [ + "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", + "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" + ]} + + # Sign a valid cert + user_content["cert_sign"] = CryptBitcoin.sign("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % ( + user_content["cert_auth_type"], + user_content["cert_user_id"].split("@")[0] + ), cert_priv) + + # Verify cert + assert site.content_manager.verifyCert("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) + + # Verify if the cert is valid for other address + assert not site.content_manager.verifyCert("data/users/badaddress/content.json", user_content) + + # Sign user content + signed_content = site.content_manager.sign( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False + ) + + # Test user cert + assert site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + + # Test banned user + cert_user_id = user_content["cert_user_id"] # My username + site.content_manager.contents["data/users/content.json"]["user_contents"]["permissions"][cert_user_id] = False + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + assert "Valid signs: 0/1" in str(err.value) + del site.content_manager.contents["data/users/content.json"]["user_contents"]["permissions"][cert_user_id] # Reset + + # Test invalid cert + user_content["cert_sign"] = CryptBitcoin.sign( + "badaddress#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"]), cert_priv + ) + signed_content = site.content_manager.sign( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False + ) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + assert "Invalid cert" in str(err.value) + + # Test banned user, signed by the site owner + user_content["cert_sign"] = CryptBitcoin.sign("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % ( + user_content["cert_auth_type"], + user_content["cert_user_id"].split("@")[0] + ), cert_priv) + cert_user_id = user_content["cert_user_id"] # My username + site.content_manager.contents["data/users/content.json"]["user_contents"]["permissions"][cert_user_id] = False + + site_privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT + del user_content["signs"] # Remove signs before signing + user_content["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), site_privatekey) + } + assert site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(user_content).encode()), ignore_same=False + ) + + def testMissingCert(self, site): + user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" + cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" + + user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] + rules_content = site.content_manager.contents["data/users/content.json"] + + # Override valid cert signers for the test + rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [ + "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", + "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" + ] + + # Sign a valid cert + user_content["cert_sign"] = CryptBitcoin.sign("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % ( + user_content["cert_auth_type"], + user_content["cert_user_id"].split("@")[0] + ), cert_priv) + signed_content = site.content_manager.sign( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False + ) + + assert site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + + # Test invalid cert_user_id + user_content["cert_user_id"] = "nodomain" + user_content["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), user_priv) + } + signed_content = site.content_manager.sign( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False + ) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + assert "Invalid domain in cert_user_id" in str(err.value) + + # Test removed cert + del user_content["cert_user_id"] + del user_content["cert_auth_type"] + del user_content["signs"] # Remove signs before signing + user_content["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), user_priv) + } + signed_content = site.content_manager.sign( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False + ) + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + assert "Missing cert_user_id" in str(err.value) + + + def testCertSignersPattern(self, site): + user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" + cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" # For 14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet + + user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] + rules_content = site.content_manager.contents["data/users/content.json"] + + # Override valid cert signers for the test + rules_content["user_contents"]["cert_signers_pattern"] = "14wgQ[0-9][A-Z]" + + # Sign a valid cert + user_content["cert_user_id"] = "certuser@14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" + user_content["cert_sign"] = CryptBitcoin.sign("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % ( + user_content["cert_auth_type"], + "certuser" + ), cert_priv) + signed_content = site.content_manager.sign( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False + ) + + assert site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + + # Cert does not matches the pattern + rules_content["user_contents"]["cert_signers_pattern"] = "14wgX[0-9][A-Z]" + + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + assert "Invalid cert signer: 14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" in str(err.value) + + # Removed cert_signers_pattern + del rules_content["user_contents"]["cert_signers_pattern"] + + with pytest.raises(VerifyError) as err: + site.content_manager.verifyFile( + "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False + ) + assert "Invalid cert signer: 14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" in str(err.value) + + + def testNewFile(self, site): + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT + inner_path = "data/users/1NEWrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" + + site.storage.writeJson(inner_path, {"test": "data"}) + site.content_manager.sign(inner_path, privatekey) + assert "test" in site.storage.loadJson(inner_path) + + site.storage.delete(inner_path) diff --git a/src/Test/TestCryptBitcoin.py b/src/Test/TestCryptBitcoin.py new file mode 100644 index 00000000..2bc087b5 --- /dev/null +++ b/src/Test/TestCryptBitcoin.py @@ -0,0 +1,48 @@ +from Crypt import CryptBitcoin + + +class TestCryptBitcoin: + def testSign(self, crypt_bitcoin_lib): + privatekey = "5K9S6dVpufGnroRgFrT6wsKiz2mJRYsC73eWDmajaHserAp3F1C" + privatekey_bad = "5Jbm9rrusXyApAoM8YoM4Rja337zMMoBUMRJ1uijiguU2aZRnwC" + + # Get address by privatekey + address = crypt_bitcoin_lib.privatekeyToAddress(privatekey) + assert address == "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz" + + address_bad = crypt_bitcoin_lib.privatekeyToAddress(privatekey_bad) + assert address_bad != "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz" + + # Text signing + data_len_list = list(range(0, 300, 10)) + data_len_list += [1024, 2048, 1024 * 128, 1024 * 1024, 1024 * 2048] + for data_len in data_len_list: + data = data_len * "!" + sign = crypt_bitcoin_lib.sign(data, privatekey) + + assert crypt_bitcoin_lib.verify(data, address, sign) + assert not crypt_bitcoin_lib.verify("invalid" + data, address, sign) + + # Signed by bad privatekey + sign_bad = crypt_bitcoin_lib.sign("hello", privatekey_bad) + assert not crypt_bitcoin_lib.verify("hello", address, sign_bad) + + def testVerify(self, crypt_bitcoin_lib): + sign_uncompressed = b'G6YkcFTuwKMVMHI2yycGQIFGbCZVNsZEZvSlOhKpHUt/BlADY94egmDAWdlrbbFrP9wH4aKcEfbLO8sa6f63VU0=' + assert crypt_bitcoin_lib.verify("1NQUem2M4cAqWua6BVFBADtcSP55P4QobM#web/gitcenter", "19Bir5zRm1yo4pw9uuxQL8xwf9b7jqMpR", sign_uncompressed) + + sign_compressed = b'H6YkcFTuwKMVMHI2yycGQIFGbCZVNsZEZvSlOhKpHUt/BlADY94egmDAWdlrbbFrP9wH4aKcEfbLO8sa6f63VU0=' + assert crypt_bitcoin_lib.verify("1NQUem2M4cAqWua6BVFBADtcSP55P4QobM#web/gitcenter", "1KH5BdNnqxh2KRWMMT8wUXzUgz4vVQ4S8p", sign_compressed) + + def testNewPrivatekey(self): + assert CryptBitcoin.newPrivatekey() != CryptBitcoin.newPrivatekey() + assert CryptBitcoin.privatekeyToAddress(CryptBitcoin.newPrivatekey()) + + def testNewSeed(self): + assert CryptBitcoin.newSeed() != CryptBitcoin.newSeed() + assert CryptBitcoin.privatekeyToAddress( + CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 0) + ) + assert CryptBitcoin.privatekeyToAddress( + CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 2**256) + ) diff --git a/src/Test/TestCryptConnection.py b/src/Test/TestCryptConnection.py new file mode 100644 index 00000000..46d2affc --- /dev/null +++ b/src/Test/TestCryptConnection.py @@ -0,0 +1,23 @@ +import os + +from Config import config +from Crypt import CryptConnection + + +class TestCryptConnection: + def testSslCert(self): + # Remove old certs + if os.path.isfile("%s/cert-rsa.pem" % config.data_dir): + os.unlink("%s/cert-rsa.pem" % config.data_dir) + if os.path.isfile("%s/key-rsa.pem" % config.data_dir): + os.unlink("%s/key-rsa.pem" % config.data_dir) + + # Generate certs + CryptConnection.manager.loadCerts() + + assert "tls-rsa" in CryptConnection.manager.crypt_supported + assert CryptConnection.manager.selectCrypt(["tls-rsa", "unknown"]) == "tls-rsa" # It should choose the known crypt + + # Check openssl cert generation + assert os.path.isfile("%s/cert-rsa.pem" % config.data_dir) + assert os.path.isfile("%s/key-rsa.pem" % config.data_dir) diff --git a/src/Test/TestCryptHash.py b/src/Test/TestCryptHash.py new file mode 100644 index 00000000..b91dbcca --- /dev/null +++ b/src/Test/TestCryptHash.py @@ -0,0 +1,31 @@ +import base64 + +from Crypt import CryptHash + +sha512t_sum_hex = "2e9466d8aa1f340c91203b4ddbe9b6669879616a1b8e9571058a74195937598d" +sha512t_sum_bin = b".\x94f\xd8\xaa\x1f4\x0c\x91 ;M\xdb\xe9\xb6f\x98yaj\x1b\x8e\x95q\x05\x8at\x19Y7Y\x8d" +sha256_sum_hex = "340cd04be7f530e3a7c1bc7b24f225ba5762ec7063a56e1ae01a30d56722e5c3" + + +class TestCryptBitcoin: + + def testSha(self, site): + file_path = site.storage.getPath("dbschema.json") + assert CryptHash.sha512sum(file_path) == sha512t_sum_hex + assert CryptHash.sha512sum(open(file_path, "rb")) == sha512t_sum_hex + assert CryptHash.sha512sum(open(file_path, "rb"), format="digest") == sha512t_sum_bin + + assert CryptHash.sha256sum(file_path) == sha256_sum_hex + assert CryptHash.sha256sum(open(file_path, "rb")) == sha256_sum_hex + + with open(file_path, "rb") as f: + hash = CryptHash.Sha512t(f.read(100)) + hash.hexdigest() != sha512t_sum_hex + hash.update(f.read(1024 * 1024)) + assert hash.hexdigest() == sha512t_sum_hex + + def testRandom(self): + assert len(CryptHash.random(64)) == 64 + assert CryptHash.random() != CryptHash.random() + assert bytes.fromhex(CryptHash.random(encoding="hex")) + assert base64.b64decode(CryptHash.random(encoding="base64")) diff --git a/src/Test/TestDb.py b/src/Test/TestDb.py new file mode 100644 index 00000000..67f383a3 --- /dev/null +++ b/src/Test/TestDb.py @@ -0,0 +1,137 @@ +import io + + +class TestDb: + def testCheckTables(self, db): + tables = [row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")] + assert "keyvalue" in tables # To store simple key -> value + assert "json" in tables # Json file path registry + assert "test" in tables # The table defined in dbschema.json + + # Verify test table + cols = [col["name"] for col in db.execute("PRAGMA table_info(test)")] + assert "test_id" in cols + assert "title" in cols + + # Add new table + assert "newtest" not in tables + db.schema["tables"]["newtest"] = { + "cols": [ + ["newtest_id", "INTEGER"], + ["newtitle", "TEXT"], + ], + "indexes": ["CREATE UNIQUE INDEX newtest_id ON newtest(newtest_id)"], + "schema_changed": 1426195822 + } + db.checkTables() + tables = [row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")] + assert "test" in tables + assert "newtest" in tables + + def testQueries(self, db): + # Test insert + for i in range(100): + db.execute("INSERT INTO test ?", {"test_id": i, "title": "Test #%s" % i}) + + assert db.execute("SELECT COUNT(*) AS num FROM test").fetchone()["num"] == 100 + + # Test single select + assert db.execute("SELECT COUNT(*) AS num FROM test WHERE ?", {"test_id": 1}).fetchone()["num"] == 1 + + # Test multiple select + assert db.execute("SELECT COUNT(*) AS num FROM test WHERE ?", {"test_id": [1, 2, 3]}).fetchone()["num"] == 3 + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"test_id": [1, 2, 3], "title": "Test #2"} + ).fetchone()["num"] == 1 + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"test_id": [1, 2, 3], "title": ["Test #2", "Test #3", "Test #4"]} + ).fetchone()["num"] == 2 + + # Test multiple select using named params + assert db.execute("SELECT COUNT(*) AS num FROM test WHERE test_id IN :test_id", {"test_id": [1, 2, 3]}).fetchone()["num"] == 3 + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE test_id IN :test_id AND title = :title", + {"test_id": [1, 2, 3], "title": "Test #2"} + ).fetchone()["num"] == 1 + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE test_id IN :test_id AND title IN :title", + {"test_id": [1, 2, 3], "title": ["Test #2", "Test #3", "Test #4"]} + ).fetchone()["num"] == 2 + + # Large ammount of IN values + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"not__test_id": list(range(2, 3000))} + ).fetchone()["num"] == 2 + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"test_id": list(range(50, 3000))} + ).fetchone()["num"] == 50 + + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"not__title": ["Test #%s" % i for i in range(50, 3000)]} + ).fetchone()["num"] == 50 + + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"title__like": "%20%"} + ).fetchone()["num"] == 1 + + # Test named parameter escaping + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE test_id = :test_id AND title LIKE :titlelike", + {"test_id": 1, "titlelike": "Test%"} + ).fetchone()["num"] == 1 + + def testEscaping(self, db): + # Test insert + for i in range(100): + db.execute("INSERT INTO test ?", {"test_id": i, "title": "Test '\" #%s" % i}) + + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"title": "Test '\" #1"} + ).fetchone()["num"] == 1 + + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"title": ["Test '\" #%s" % i for i in range(0, 50)]} + ).fetchone()["num"] == 50 + + assert db.execute( + "SELECT COUNT(*) AS num FROM test WHERE ?", + {"not__title": ["Test '\" #%s" % i for i in range(50, 3000)]} + ).fetchone()["num"] == 50 + + + def testUpdateJson(self, db): + f = io.BytesIO() + f.write(""" + { + "test": [ + {"test_id": 1, "title": "Test 1 title", "extra col": "Ignore it"} + ] + } + """.encode()) + f.seek(0) + assert db.updateJson(db.db_dir + "data.json", f) is True + assert db.execute("SELECT COUNT(*) AS num FROM test_importfilter").fetchone()["num"] == 1 + assert db.execute("SELECT COUNT(*) AS num FROM test").fetchone()["num"] == 1 + + def testUnsafePattern(self, db): + db.schema["maps"] = {"[A-Za-z.]*": db.schema["maps"]["data.json"]} # Only repetition of . supported + f = io.StringIO() + f.write(""" + { + "test": [ + {"test_id": 1, "title": "Test 1 title", "extra col": "Ignore it"} + ] + } + """) + f.seek(0) + assert db.updateJson(db.db_dir + "data.json", f) is False + assert db.execute("SELECT COUNT(*) AS num FROM test_importfilter").fetchone()["num"] == 0 + assert db.execute("SELECT COUNT(*) AS num FROM test").fetchone()["num"] == 0 diff --git a/src/Test/TestDbQuery.py b/src/Test/TestDbQuery.py new file mode 100644 index 00000000..597bc950 --- /dev/null +++ b/src/Test/TestDbQuery.py @@ -0,0 +1,31 @@ +import re + +from Db.DbQuery import DbQuery + + +class TestDbQuery: + def testParse(self): + query_text = """ + SELECT + 'comment' AS type, + date_added, post.title AS title, + keyvalue.value || ': ' || comment.body AS body, + '?Post:' || comment.post_id || '#Comments' AS url + FROM + comment + LEFT JOIN json USING (json_id) + LEFT JOIN json AS json_content ON (json_content.directory = json.directory AND json_content.file_name='content.json') + LEFT JOIN keyvalue ON (keyvalue.json_id = json_content.json_id AND key = 'cert_user_id') + LEFT JOIN post ON (comment.post_id = post.post_id) + WHERE + post.date_added > 123 + ORDER BY + date_added DESC + LIMIT 20 + """ + query = DbQuery(query_text) + assert query.parts["LIMIT"] == "20" + assert query.fields["body"] == "keyvalue.value || ': ' || comment.body" + assert re.sub("[ \r\n]", "", str(query)) == re.sub("[ \r\n]", "", query_text) + query.wheres.append("body LIKE '%hello%'") + assert "body LIKE '%hello%'" in str(query) diff --git a/src/Test/TestDebug.py b/src/Test/TestDebug.py new file mode 100644 index 00000000..e3eb20b3 --- /dev/null +++ b/src/Test/TestDebug.py @@ -0,0 +1,52 @@ +from Debug import Debug +import gevent +import os +import re + +import pytest + + +class TestDebug: + @pytest.mark.parametrize("items,expected", [ + (["@/src/A/B/C.py:17"], ["A/B/C.py line 17"]), # basic test + (["@/src/Db/Db.py:17"], ["Db.py line 17"]), # path compression + (["%s:1" % __file__], ["TestDebug.py line 1"]), + (["@/plugins/Chart/ChartDb.py:100"], ["ChartDb.py line 100"]), # plugins + (["@/main.py:17"], ["main.py line 17"]), # root + (["@\\src\\Db\\__init__.py:17"], ["Db/__init__.py line 17"]), # Windows paths + ([":1"], []), # importlib builtins + ([":1"], []), # importlib builtins + (["/home/ivanq/ZeroNet/src/main.py:13"], ["?/src/main.py line 13"]), # best-effort anonymization + (["C:\\ZeroNet\\core\\src\\main.py:13"], ["?/src/main.py line 13"]), + (["/root/main.py:17"], ["/root/main.py line 17"]), + (["{gevent}:13"], ["/__init__.py line 13"]), # modules + (["{os}:13"], [" line 13"]), # python builtin modules + (["src/gevent/event.py:17"], ["/event.py line 17"]), # gevent-overriden __file__ + (["@/src/Db/Db.py:17", "@/src/Db/DbQuery.py:1"], ["Db.py line 17", "DbQuery.py line 1"]), # mutliple args + (["@/src/Db/Db.py:17", "@/src/Db/Db.py:1"], ["Db.py line 17", "1"]), # same file + (["{os}:1", "@/src/Db/Db.py:17"], [" line 1", "Db.py line 17"]), # builtins + (["{gevent}:1"] + ["{os}:3"] * 4 + ["@/src/Db/Db.py:17"], ["/__init__.py line 1", "...", "Db.py line 17"]) + ]) + def testFormatTraceback(self, items, expected): + q_items = [] + for item in items: + file, line = item.rsplit(":", 1) + if file.startswith("@"): + file = Debug.root_dir + file[1:] + file = file.replace("{os}", os.__file__) + file = file.replace("{gevent}", gevent.__file__) + q_items.append((file, int(line))) + assert Debug.formatTraceback(q_items) == expected + + def testFormatException(self): + try: + raise ValueError("Test exception") + except Exception: + assert re.match(r"ValueError: Test exception in TestDebug.py line [0-9]+", Debug.formatException()) + try: + os.path.abspath(1) + except Exception: + assert re.search(r"in TestDebug.py line [0-9]+ > <(posixpath|ntpath)> line ", Debug.formatException()) + + def testFormatStack(self): + assert re.match(r"TestDebug.py line [0-9]+ > <_pytest>/python.py line [0-9]+", Debug.formatStack()) diff --git a/src/Test/TestDiff.py b/src/Test/TestDiff.py new file mode 100644 index 00000000..622951a1 --- /dev/null +++ b/src/Test/TestDiff.py @@ -0,0 +1,58 @@ +import io + +from util import Diff + + +class TestDiff: + def testDiff(self): + assert Diff.diff( + [], + ["one", "two", "three"] + ) == [("+", ["one", "two","three"])] + + assert Diff.diff( + ["one", "two", "three"], + ["one", "two", "three", "four", "five"] + ) == [("=", 11), ("+", ["four", "five"])] + + assert Diff.diff( + ["one", "two", "three", "six"], + ["one", "two", "three", "four", "five", "six"] + ) == [("=", 11), ("+", ["four", "five"]), ("=", 3)] + + assert Diff.diff( + ["one", "two", "three", "hmm", "six"], + ["one", "two", "three", "four", "five", "six"] + ) == [("=", 11), ("-", 3), ("+", ["four", "five"]), ("=", 3)] + + assert Diff.diff( + ["one", "two", "three"], + [] + ) == [("-", 11)] + + def testUtf8(self): + assert Diff.diff( + ["one", "\xe5\xad\xa6\xe4\xb9\xa0\xe4\xb8\x8b", "two", "three"], + ["one", "\xe5\xad\xa6\xe4\xb9\xa0\xe4\xb8\x8b", "two", "three", "four", "five"] + ) == [("=", 20), ("+", ["four", "five"])] + + def testDiffLimit(self): + old_f = io.BytesIO(b"one\ntwo\nthree\nhmm\nsix") + new_f = io.BytesIO(b"one\ntwo\nthree\nfour\nfive\nsix") + actions = Diff.diff(list(old_f), list(new_f), limit=1024) + assert actions + + old_f = io.BytesIO(b"one\ntwo\nthree\nhmm\nsix") + new_f = io.BytesIO(b"one\ntwo\nthree\nfour\nfive\nsix"*1024) + actions = Diff.diff(list(old_f), list(new_f), limit=1024) + assert actions is False + + def testPatch(self): + old_f = io.BytesIO(b"one\ntwo\nthree\nhmm\nsix") + new_f = io.BytesIO(b"one\ntwo\nthree\nfour\nfive\nsix") + actions = Diff.diff( + list(old_f), + list(new_f) + ) + old_f.seek(0) + assert Diff.patch(old_f, actions).getvalue() == new_f.getvalue() diff --git a/src/Test/TestEvent.py b/src/Test/TestEvent.py new file mode 100644 index 00000000..8bdafaaa --- /dev/null +++ b/src/Test/TestEvent.py @@ -0,0 +1,65 @@ +import util + + +class ExampleClass(object): + def __init__(self): + self.called = [] + self.onChanged = util.Event() + + def increment(self, title): + self.called.append(title) + + +class TestEvent: + def testEvent(self): + test_obj = ExampleClass() + test_obj.onChanged.append(lambda: test_obj.increment("Called #1")) + test_obj.onChanged.append(lambda: test_obj.increment("Called #2")) + test_obj.onChanged.once(lambda: test_obj.increment("Once")) + + assert test_obj.called == [] + test_obj.onChanged() + assert test_obj.called == ["Called #1", "Called #2", "Once"] + test_obj.onChanged() + test_obj.onChanged() + assert test_obj.called == ["Called #1", "Called #2", "Once", "Called #1", "Called #2", "Called #1", "Called #2"] + + def testOnce(self): + test_obj = ExampleClass() + test_obj.onChanged.once(lambda: test_obj.increment("Once test #1")) + + # It should be called only once + assert test_obj.called == [] + test_obj.onChanged() + assert test_obj.called == ["Once test #1"] + test_obj.onChanged() + test_obj.onChanged() + assert test_obj.called == ["Once test #1"] + + def testOnceMultiple(self): + test_obj = ExampleClass() + # Allow queue more than once + test_obj.onChanged.once(lambda: test_obj.increment("Once test #1")) + test_obj.onChanged.once(lambda: test_obj.increment("Once test #2")) + test_obj.onChanged.once(lambda: test_obj.increment("Once test #3")) + + assert test_obj.called == [] + test_obj.onChanged() + assert test_obj.called == ["Once test #1", "Once test #2", "Once test #3"] + test_obj.onChanged() + test_obj.onChanged() + assert test_obj.called == ["Once test #1", "Once test #2", "Once test #3"] + + def testOnceNamed(self): + test_obj = ExampleClass() + # Dont store more that one from same type + test_obj.onChanged.once(lambda: test_obj.increment("Once test #1/1"), "type 1") + test_obj.onChanged.once(lambda: test_obj.increment("Once test #1/2"), "type 1") + test_obj.onChanged.once(lambda: test_obj.increment("Once test #2"), "type 2") + + assert test_obj.called == [] + test_obj.onChanged() + assert test_obj.called == ["Once test #1/1", "Once test #2"] + test_obj.onChanged() + test_obj.onChanged() + assert test_obj.called == ["Once test #1/1", "Once test #2"] diff --git a/src/Test/TestFileRequest.py b/src/Test/TestFileRequest.py new file mode 100644 index 00000000..3fabc271 --- /dev/null +++ b/src/Test/TestFileRequest.py @@ -0,0 +1,124 @@ +import io + +import pytest +import time + +from Connection import ConnectionServer +from Connection import Connection +from File import FileServer + + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestFileRequest: + def testGetFile(self, file_server, site): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer(file_server.ip, 1545) + + connection = client.getConnection(file_server.ip, 1544) + file_server.sites[site.address] = site + + # Normal request + response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0}) + assert b"sign" in response["body"] + + response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0, "file_size": site.storage.getSize("content.json")}) + assert b"sign" in response["body"] + + # Invalid file + response = connection.request("getFile", {"site": site.address, "inner_path": "invalid.file", "location": 0}) + assert "File read error" in response["error"] + + # Location over size + response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 1024 * 1024}) + assert "File read error" in response["error"] + + # Stream from parent dir + response = connection.request("getFile", {"site": site.address, "inner_path": "../users.json", "location": 0}) + assert "File read exception" in response["error"] + + # Invalid site + response = connection.request("getFile", {"site": "", "inner_path": "users.json", "location": 0}) + assert "Unknown site" in response["error"] + + response = connection.request("getFile", {"site": ".", "inner_path": "users.json", "location": 0}) + assert "Unknown site" in response["error"] + + # Invalid size + response = connection.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0, "file_size": 1234}) + assert "File size does not match" in response["error"] + + # Invalid path + for path in ["../users.json", "./../users.json", "data/../content.json", ".../users.json"]: + for sep in ["/", "\\"]: + response = connection.request("getFile", {"site": site.address, "inner_path": path.replace("/", sep), "location": 0}) + assert response["error"] == 'File read exception' + + connection.close() + client.stop() + + def testStreamFile(self, file_server, site): + file_server.ip_incoming = {} # Reset flood protection + client = ConnectionServer(file_server.ip, 1545) + connection = client.getConnection(file_server.ip, 1544) + file_server.sites[site.address] = site + + buff = io.BytesIO() + response = connection.request("streamFile", {"site": site.address, "inner_path": "content.json", "location": 0}, buff) + assert "stream_bytes" in response + assert b"sign" in buff.getvalue() + + # Invalid file + buff = io.BytesIO() + response = connection.request("streamFile", {"site": site.address, "inner_path": "invalid.file", "location": 0}, buff) + assert "File read error" in response["error"] + + # Location over size + buff = io.BytesIO() + response = connection.request( + "streamFile", {"site": site.address, "inner_path": "content.json", "location": 1024 * 1024}, buff + ) + assert "File read error" in response["error"] + + # Stream from parent dir + buff = io.BytesIO() + response = connection.request("streamFile", {"site": site.address, "inner_path": "../users.json", "location": 0}, buff) + assert "File read exception" in response["error"] + + connection.close() + client.stop() + + def testPex(self, file_server, site, site_temp): + file_server.sites[site.address] = site + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + connection = client.getConnection(file_server.ip, 1544) + + # Add new fake peer to site + fake_peer = site.addPeer(file_server.ip_external, 11337, return_peer=True) + # Add fake connection to it + fake_peer.connection = Connection(file_server, file_server.ip_external, 11337) + fake_peer.connection.last_recv_time = time.time() + assert fake_peer in site.getConnectablePeers() + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer(file_server.ip, 1544) + + assert "%s:11337" % file_server.ip_external not in site_temp.peers + assert peer_file_server.pex() + assert "%s:11337" % file_server.ip_external in site_temp.peers + + # Should not exchange private peers from local network + fake_peer_private = site.addPeer("192.168.0.1", 11337, return_peer=True) + assert fake_peer_private not in site.getConnectablePeers(allow_private=False) + fake_peer_private.connection = Connection(file_server, "192.168.0.1", 11337) + fake_peer_private.connection.last_recv_time = time.time() + + assert "192.168.0.1:11337" not in site_temp.peers + assert not peer_file_server.pex() + assert "192.168.0.1:11337" not in site_temp.peers + + + connection.close() + client.stop() diff --git a/src/Test/TestFlag.py b/src/Test/TestFlag.py new file mode 100644 index 00000000..12fd8165 --- /dev/null +++ b/src/Test/TestFlag.py @@ -0,0 +1,39 @@ +import os + +import pytest + +from util.Flag import Flag + +class TestFlag: + def testFlagging(self): + flag = Flag() + @flag.admin + @flag.no_multiuser + def testFn(anything): + return anything + + assert "admin" in flag.db["testFn"] + assert "no_multiuser" in flag.db["testFn"] + + def testSubclassedFlagging(self): + flag = Flag() + class Test: + @flag.admin + @flag.no_multiuser + def testFn(anything): + return anything + + class SubTest(Test): + pass + + assert "admin" in flag.db["testFn"] + assert "no_multiuser" in flag.db["testFn"] + + def testInvalidFlag(self): + flag = Flag() + with pytest.raises(Exception) as err: + @flag.no_multiuser + @flag.unknown_flag + def testFn(anything): + return anything + assert "Invalid flag" in str(err.value) diff --git a/src/Test/TestHelper.py b/src/Test/TestHelper.py new file mode 100644 index 00000000..07644ec0 --- /dev/null +++ b/src/Test/TestHelper.py @@ -0,0 +1,79 @@ +import socket +import struct +import os + +import pytest +from util import helper +from Config import config + + +@pytest.mark.usefixtures("resetSettings") +class TestHelper: + def testShellquote(self): + assert helper.shellquote("hel'lo") == "\"hel'lo\"" # Allow ' + assert helper.shellquote('hel"lo') == '"hello"' # Remove " + assert helper.shellquote("hel'lo", 'hel"lo') == ('"hel\'lo"', '"hello"') + + def testPackAddress(self): + for port in [1, 1000, 65535]: + for ip in ["1.1.1.1", "127.0.0.1", "0.0.0.0", "255.255.255.255", "192.168.1.1"]: + assert len(helper.packAddress(ip, port)) == 6 + assert helper.unpackAddress(helper.packAddress(ip, port)) == (ip, port) + + for ip in ["1:2:3:4:5:6:7:8", "::1", "2001:19f0:6c01:e76:5400:1ff:fed6:3eca", "2001:4860:4860::8888"]: + assert len(helper.packAddress(ip, port)) == 18 + assert helper.unpackAddress(helper.packAddress(ip, port)) == (ip, port) + + assert len(helper.packOnionAddress("boot3rdez4rzn36x.onion", port)) == 12 + assert helper.unpackOnionAddress(helper.packOnionAddress("boot3rdez4rzn36x.onion", port)) == ("boot3rdez4rzn36x.onion", port) + + with pytest.raises(struct.error): + helper.packAddress("1.1.1.1", 100000) + + with pytest.raises(socket.error): + helper.packAddress("999.1.1.1", 1) + + with pytest.raises(Exception): + helper.unpackAddress("X") + + def testGetDirname(self): + assert helper.getDirname("data/users/content.json") == "data/users/" + assert helper.getDirname("data/users") == "data/" + assert helper.getDirname("") == "" + assert helper.getDirname("content.json") == "" + assert helper.getDirname("data/users/") == "data/users/" + assert helper.getDirname("/data/users/content.json") == "data/users/" + + def testGetFilename(self): + assert helper.getFilename("data/users/content.json") == "content.json" + assert helper.getFilename("data/users") == "users" + assert helper.getFilename("") == "" + assert helper.getFilename("content.json") == "content.json" + assert helper.getFilename("data/users/") == "" + assert helper.getFilename("/data/users/content.json") == "content.json" + + def testIsIp(self): + assert helper.isIp("1.2.3.4") + assert helper.isIp("255.255.255.255") + assert not helper.isIp("any.host") + assert not helper.isIp("1.2.3.4.com") + assert not helper.isIp("1.2.3.4.any.host") + + def testIsPrivateIp(self): + assert helper.isPrivateIp("192.168.1.1") + assert not helper.isPrivateIp("1.1.1.1") + assert helper.isPrivateIp("fe80::44f0:3d0:4e6:637c") + assert not helper.isPrivateIp("fca5:95d6:bfde:d902:8951:276e:1111:a22c") # cjdns + + def testOpenLocked(self): + locked_f = helper.openLocked(config.data_dir + "/locked.file") + assert locked_f + with pytest.raises(BlockingIOError): + locked_f_again = helper.openLocked(config.data_dir + "/locked.file") + locked_f_different = helper.openLocked(config.data_dir + "/locked_different.file") + + locked_f.close() + locked_f_different.close() + + os.unlink(locked_f.name) + os.unlink(locked_f_different.name) diff --git a/src/Test/TestMsgpack.py b/src/Test/TestMsgpack.py new file mode 100644 index 00000000..5a0b6d4d --- /dev/null +++ b/src/Test/TestMsgpack.py @@ -0,0 +1,88 @@ +import io +import os + +import msgpack +import pytest + +from Config import config +from util import Msgpack +from collections import OrderedDict + + +class TestMsgpack: + test_data = OrderedDict( + sorted({"cmd": "fileGet", "bin": b'p\x81zDhL\xf0O\xd0\xaf', "params": {"site": "1Site"}, "utf8": b'\xc3\xa1rv\xc3\xadzt\xc5\xb1r\xc5\x91'.decode("utf8"), "list": [b'p\x81zDhL\xf0O\xd0\xaf', b'p\x81zDhL\xf0O\xd0\xaf']}.items()) + ) + + def testPacking(self): + assert Msgpack.pack(self.test_data) == b'\x85\xa3bin\xc4\np\x81zDhL\xf0O\xd0\xaf\xa3cmd\xa7fileGet\xa4list\x92\xc4\np\x81zDhL\xf0O\xd0\xaf\xc4\np\x81zDhL\xf0O\xd0\xaf\xa6params\x81\xa4site\xa51Site\xa4utf8\xad\xc3\xa1rv\xc3\xadzt\xc5\xb1r\xc5\x91' + assert Msgpack.pack(self.test_data, use_bin_type=False) == b'\x85\xa3bin\xaap\x81zDhL\xf0O\xd0\xaf\xa3cmd\xa7fileGet\xa4list\x92\xaap\x81zDhL\xf0O\xd0\xaf\xaap\x81zDhL\xf0O\xd0\xaf\xa6params\x81\xa4site\xa51Site\xa4utf8\xad\xc3\xa1rv\xc3\xadzt\xc5\xb1r\xc5\x91' + + def testUnpackinkg(self): + assert Msgpack.unpack(Msgpack.pack(self.test_data)) == self.test_data + + @pytest.mark.parametrize("unpacker_class", [msgpack.Unpacker, msgpack.fallback.Unpacker]) + def testUnpacker(self, unpacker_class): + unpacker = unpacker_class(raw=False) + + data = msgpack.packb(self.test_data, use_bin_type=True) + data += msgpack.packb(self.test_data, use_bin_type=True) + + messages = [] + for char in data: + unpacker.feed(bytes([char])) + for message in unpacker: + messages.append(message) + + assert len(messages) == 2 + assert messages[0] == self.test_data + assert messages[0] == messages[1] + + def testStreaming(self): + bin_data = os.urandom(20) + f = Msgpack.FilePart("%s/users.json" % config.data_dir, "rb") + f.read_bytes = 30 + + data = {"cmd": "response", "body": f, "bin": bin_data} + + out_buff = io.BytesIO() + Msgpack.stream(data, out_buff.write) + out_buff.seek(0) + + data_packb = { + "cmd": "response", + "body": open("%s/users.json" % config.data_dir, "rb").read(30), + "bin": bin_data + } + + out_buff.seek(0) + data_unpacked = Msgpack.unpack(out_buff.read()) + assert data_unpacked == data_packb + assert data_unpacked["cmd"] == "response" + assert type(data_unpacked["body"]) == bytes + + def testBackwardCompatibility(self): + packed = {} + packed["py3"] = Msgpack.pack(self.test_data, use_bin_type=False) + packed["py3_bin"] = Msgpack.pack(self.test_data, use_bin_type=True) + for key, val in packed.items(): + unpacked = Msgpack.unpack(val) + type(unpacked["utf8"]) == str + type(unpacked["bin"]) == bytes + + # Packed with use_bin_type=False (pre-ZeroNet 0.7.0) + unpacked = Msgpack.unpack(packed["py3"], decode=True) + type(unpacked["utf8"]) == str + type(unpacked["bin"]) == bytes + assert len(unpacked["utf8"]) == 9 + assert len(unpacked["bin"]) == 10 + with pytest.raises(UnicodeDecodeError) as err: # Try to decode binary as utf-8 + unpacked = Msgpack.unpack(packed["py3"], decode=False) + + # Packed with use_bin_type=True + unpacked = Msgpack.unpack(packed["py3_bin"], decode=False) + type(unpacked["utf8"]) == str + type(unpacked["bin"]) == bytes + assert len(unpacked["utf8"]) == 9 + assert len(unpacked["bin"]) == 10 + diff --git a/src/Test/TestNoparallel.py b/src/Test/TestNoparallel.py new file mode 100644 index 00000000..6fc4f57d --- /dev/null +++ b/src/Test/TestNoparallel.py @@ -0,0 +1,167 @@ +import time + +import gevent +import pytest + +import util +from util import ThreadPool + + +@pytest.fixture(params=['gevent.spawn', 'thread_pool.spawn']) +def queue_spawn(request): + thread_pool = ThreadPool.ThreadPool(10) + if request.param == "gevent.spawn": + return gevent.spawn + else: + return thread_pool.spawn + + +class ExampleClass(object): + def __init__(self): + self.counted = 0 + + @util.Noparallel() + def countBlocking(self, num=5): + for i in range(1, num + 1): + time.sleep(0.1) + self.counted += 1 + return "counted:%s" % i + + @util.Noparallel(queue=True, ignore_class=True) + def countQueue(self, num=5): + for i in range(1, num + 1): + time.sleep(0.1) + self.counted += 1 + return "counted:%s" % i + + @util.Noparallel(blocking=False) + def countNoblocking(self, num=5): + for i in range(1, num + 1): + time.sleep(0.01) + self.counted += 1 + return "counted:%s" % i + + +class TestNoparallel: + def testBlocking(self, queue_spawn): + obj1 = ExampleClass() + obj2 = ExampleClass() + + # Dont allow to call again until its running and wait until its running + threads = [ + queue_spawn(obj1.countBlocking), + queue_spawn(obj1.countBlocking), + queue_spawn(obj1.countBlocking), + queue_spawn(obj2.countBlocking) + ] + assert obj2.countBlocking() == "counted:5" # The call is ignored as obj2.countBlocking already counting, but block until its finishes + gevent.joinall(threads) + assert [thread.value for thread in threads] == ["counted:5", "counted:5", "counted:5", "counted:5"] + obj2.countBlocking() # Allow to call again as obj2.countBlocking finished + + assert obj1.counted == 5 + assert obj2.counted == 10 + + def testNoblocking(self): + obj1 = ExampleClass() + + thread1 = obj1.countNoblocking() + thread2 = obj1.countNoblocking() # Ignored + + assert obj1.counted == 0 + time.sleep(0.1) + assert thread1.value == "counted:5" + assert thread2.value == "counted:5" + assert obj1.counted == 5 + + obj1.countNoblocking().join() # Allow again and wait until finishes + assert obj1.counted == 10 + + def testQueue(self, queue_spawn): + obj1 = ExampleClass() + + queue_spawn(obj1.countQueue, num=1) + queue_spawn(obj1.countQueue, num=1) + queue_spawn(obj1.countQueue, num=1) + + time.sleep(0.3) + assert obj1.counted == 2 # No multi-queue supported + + obj2 = ExampleClass() + queue_spawn(obj2.countQueue, num=10) + queue_spawn(obj2.countQueue, num=10) + + time.sleep(1.5) # Call 1 finished, call 2 still working + assert 10 < obj2.counted < 20 + + queue_spawn(obj2.countQueue, num=10) + time.sleep(2.0) + + assert obj2.counted == 30 + + def testQueueOverload(self): + obj1 = ExampleClass() + + threads = [] + for i in range(1000): + thread = gevent.spawn(obj1.countQueue, num=5) + threads.append(thread) + + gevent.joinall(threads) + assert obj1.counted == 5 * 2 # Only called twice (no multi-queue allowed) + + def testIgnoreClass(self, queue_spawn): + obj1 = ExampleClass() + obj2 = ExampleClass() + + threads = [ + queue_spawn(obj1.countQueue), + queue_spawn(obj1.countQueue), + queue_spawn(obj1.countQueue), + queue_spawn(obj2.countQueue), + queue_spawn(obj2.countQueue) + ] + s = time.time() + time.sleep(0.001) + gevent.joinall(threads) + + # Queue limited to 2 calls (every call takes counts to 5 and takes 0.05 sec) + assert obj1.counted + obj2.counted == 10 + + taken = time.time() - s + assert 1.2 > taken >= 1.0 # 2 * 0.5s count = ~1s + + def testException(self, queue_spawn): + class MyException(Exception): + pass + + @util.Noparallel() + def raiseException(): + raise MyException("Test error!") + + with pytest.raises(MyException) as err: + raiseException() + assert str(err.value) == "Test error!" + + with pytest.raises(MyException) as err: + queue_spawn(raiseException).get() + assert str(err.value) == "Test error!" + + def testMultithreadMix(self, queue_spawn): + obj1 = ExampleClass() + with ThreadPool.ThreadPool(10) as thread_pool: + s = time.time() + t1 = queue_spawn(obj1.countBlocking, 5) + time.sleep(0.01) + t2 = thread_pool.spawn(obj1.countBlocking, 5) + time.sleep(0.01) + t3 = thread_pool.spawn(obj1.countBlocking, 5) + time.sleep(0.3) + t4 = gevent.spawn(obj1.countBlocking, 5) + threads = [t1, t2, t3, t4] + for thread in threads: + assert thread.get() == "counted:5" + + time_taken = time.time() - s + assert obj1.counted == 5 + assert 0.5 < time_taken < 0.7 diff --git a/src/Test/TestPeer.py b/src/Test/TestPeer.py new file mode 100644 index 00000000..f57e046e --- /dev/null +++ b/src/Test/TestPeer.py @@ -0,0 +1,159 @@ +import time +import io + +import pytest + +from File import FileServer +from File import FileRequest +from Crypt import CryptHash +from . import Spy + + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestPeer: + def testPing(self, file_server, site, site_temp): + file_server.sites[site.address] = site + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + connection = client.getConnection(file_server.ip, 1544) + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer(file_server.ip, 1544) + + assert peer_file_server.ping() is not None + + assert peer_file_server in site_temp.peers.values() + peer_file_server.remove() + assert peer_file_server not in site_temp.peers.values() + + connection.close() + client.stop() + + def testDownloadFile(self, file_server, site, site_temp): + file_server.sites[site.address] = site + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + connection = client.getConnection(file_server.ip, 1544) + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer(file_server.ip, 1544) + + # Testing streamFile + buff = peer_file_server.getFile(site_temp.address, "content.json", streaming=True) + assert b"sign" in buff.getvalue() + + # Testing getFile + buff = peer_file_server.getFile(site_temp.address, "content.json") + assert b"sign" in buff.getvalue() + + connection.close() + client.stop() + + def testHashfield(self, site): + sample_hash = list(site.content_manager.contents["content.json"]["files_optional"].values())[0]["sha512"] + + site.storage.verifyFiles(quick_check=True) # Find what optional files we have + + # Check if hashfield has any files + assert site.content_manager.hashfield + assert len(site.content_manager.hashfield) > 0 + + # Check exsist hash + assert site.content_manager.hashfield.getHashId(sample_hash) in site.content_manager.hashfield + + # Add new hash + new_hash = CryptHash.sha512sum(io.BytesIO(b"hello")) + assert site.content_manager.hashfield.getHashId(new_hash) not in site.content_manager.hashfield + assert site.content_manager.hashfield.appendHash(new_hash) + assert not site.content_manager.hashfield.appendHash(new_hash) # Don't add second time + assert site.content_manager.hashfield.getHashId(new_hash) in site.content_manager.hashfield + + # Remove new hash + assert site.content_manager.hashfield.removeHash(new_hash) + assert site.content_manager.hashfield.getHashId(new_hash) not in site.content_manager.hashfield + + def testHashfieldExchange(self, file_server, site, site_temp): + server1 = file_server + server1.sites[site.address] = site + site.connection_server = server1 + + server2 = FileServer(file_server.ip, 1545) + server2.sites[site_temp.address] = site_temp + site_temp.connection_server = server2 + site.storage.verifyFiles(quick_check=True) # Find what optional files we have + + # Add file_server as peer to client + server2_peer1 = site_temp.addPeer(file_server.ip, 1544) + + # Check if hashfield has any files + assert len(site.content_manager.hashfield) > 0 + + # Testing hashfield sync + assert len(server2_peer1.hashfield) == 0 + assert server2_peer1.updateHashfield() # Query hashfield from peer + assert len(server2_peer1.hashfield) > 0 + + # Test force push new hashfield + site_temp.content_manager.hashfield.appendHash("AABB") + server1_peer2 = site.addPeer(file_server.ip, 1545, return_peer=True) + with Spy.Spy(FileRequest, "route") as requests: + assert len(server1_peer2.hashfield) == 0 + server2_peer1.sendMyHashfield() + assert len(server1_peer2.hashfield) == 1 + server2_peer1.sendMyHashfield() # Hashfield not changed, should be ignored + + assert len(requests) == 1 + + time.sleep(0.01) # To make hashfield change date different + + site_temp.content_manager.hashfield.appendHash("AACC") + server2_peer1.sendMyHashfield() # Push hashfield + + assert len(server1_peer2.hashfield) == 2 + assert len(requests) == 2 + + site_temp.content_manager.hashfield.appendHash("AADD") + + assert server1_peer2.updateHashfield(force=True) # Request hashfield + assert len(server1_peer2.hashfield) == 3 + assert len(requests) == 3 + + assert not server2_peer1.sendMyHashfield() # Not changed, should be ignored + assert len(requests) == 3 + + server2.stop() + + def testFindHash(self, file_server, site, site_temp): + file_server.sites[site.address] = site + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer(file_server.ip, 1544) + + assert peer_file_server.findHashIds([1234]) == {} + + # Add fake peer with requred hash + fake_peer_1 = site.addPeer(file_server.ip_external, 1544) + fake_peer_1.hashfield.append(1234) + fake_peer_2 = site.addPeer("1.2.3.5", 1545) + fake_peer_2.hashfield.append(1234) + fake_peer_2.hashfield.append(1235) + fake_peer_3 = site.addPeer("1.2.3.6", 1546) + fake_peer_3.hashfield.append(1235) + fake_peer_3.hashfield.append(1236) + + res = peer_file_server.findHashIds([1234, 1235]) + assert sorted(res[1234]) == sorted([(file_server.ip_external, 1544), ("1.2.3.5", 1545)]) + assert sorted(res[1235]) == sorted([("1.2.3.5", 1545), ("1.2.3.6", 1546)]) + + # Test my address adding + site.content_manager.hashfield.append(1234) + + res = peer_file_server.findHashIds([1234, 1235]) + assert sorted(res[1234]) == sorted([(file_server.ip_external, 1544), ("1.2.3.5", 1545), (file_server.ip, 1544)]) + assert sorted(res[1235]) == sorted([("1.2.3.5", 1545), ("1.2.3.6", 1546)]) diff --git a/src/Test/TestRateLimit.py b/src/Test/TestRateLimit.py new file mode 100644 index 00000000..fafa5f1a --- /dev/null +++ b/src/Test/TestRateLimit.py @@ -0,0 +1,100 @@ +import time + +import gevent + +from util import RateLimit + + +# Time is around limit +/- 0.05 sec +def around(t, limit): + return t >= limit - 0.05 and t <= limit + 0.05 + + +class ExampleClass(object): + def __init__(self): + self.counted = 0 + self.last_called = None + + def count(self, back="counted"): + self.counted += 1 + self.last_called = back + return back + + +class TestRateLimit: + def testCall(self): + obj1 = ExampleClass() + obj2 = ExampleClass() + + s = time.time() + assert RateLimit.call("counting", allowed_again=0.1, func=obj1.count) == "counted" + assert around(time.time() - s, 0.0) # First allow to call instantly + assert obj1.counted == 1 + + # Call again + assert not RateLimit.isAllowed("counting", 0.1) + assert RateLimit.isAllowed("something else", 0.1) + assert RateLimit.call("counting", allowed_again=0.1, func=obj1.count) == "counted" + assert around(time.time() - s, 0.1) # Delays second call within interval + assert obj1.counted == 2 + time.sleep(0.1) # Wait the cooldown time + + # Call 3 times async + s = time.time() + assert obj2.counted == 0 + threads = [ + gevent.spawn(lambda: RateLimit.call("counting", allowed_again=0.1, func=obj2.count)), # Instant + gevent.spawn(lambda: RateLimit.call("counting", allowed_again=0.1, func=obj2.count)), # 0.1s delay + gevent.spawn(lambda: RateLimit.call("counting", allowed_again=0.1, func=obj2.count)) # 0.2s delay + ] + gevent.joinall(threads) + assert [thread.value for thread in threads] == ["counted", "counted", "counted"] + assert around(time.time() - s, 0.2) + + # Wait 0.1s cooldown + assert not RateLimit.isAllowed("counting", 0.1) + time.sleep(0.11) + assert RateLimit.isAllowed("counting", 0.1) + + # No queue = instant again + s = time.time() + assert RateLimit.isAllowed("counting", 0.1) + assert RateLimit.call("counting", allowed_again=0.1, func=obj2.count) == "counted" + assert around(time.time() - s, 0.0) + + assert obj2.counted == 4 + + def testCallAsync(self): + obj1 = ExampleClass() + obj2 = ExampleClass() + + s = time.time() + RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #1").join() + assert obj1.counted == 1 # First instant + assert around(time.time() - s, 0.0) + + # After that the calls delayed + s = time.time() + t1 = RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #2") # Dumped by the next call + time.sleep(0.03) + t2 = RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #3") # Dumped by the next call + time.sleep(0.03) + t3 = RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #4") # Will be called + assert obj1.counted == 1 # Delay still in progress: Not called yet + t3.join() + assert t3.value == "call #4" + assert around(time.time() - s, 0.1) + + # Only the last one called + assert obj1.counted == 2 + assert obj1.last_called == "call #4" + + # Just called, not allowed again + assert not RateLimit.isAllowed("counting async", 0.1) + s = time.time() + t4 = RateLimit.callAsync("counting async", allowed_again=0.1, func=obj1.count, back="call #5").join() + assert obj1.counted == 3 + assert around(time.time() - s, 0.1) + assert not RateLimit.isAllowed("counting async", 0.1) + time.sleep(0.11) + assert RateLimit.isAllowed("counting async", 0.1) diff --git a/src/Test/TestSafeRe.py b/src/Test/TestSafeRe.py new file mode 100644 index 00000000..429bde50 --- /dev/null +++ b/src/Test/TestSafeRe.py @@ -0,0 +1,24 @@ +from util import SafeRe + +import pytest + + +class TestSafeRe: + def testSafeMatch(self): + assert SafeRe.match( + "((js|css)/(?!all.(js|css))|data/users/.*db|data/users/.*/.*|data/archived|.*.py)", + "js/ZeroTalk.coffee" + ) + assert SafeRe.match(".+/data.json", "data/users/1J3rJ8ecnwH2EPYa6MrgZttBNc61ACFiCj/data.json") + + @pytest.mark.parametrize("pattern", ["([a-zA-Z]+)*", "(a|aa)+*", "(a|a?)+", "(.*a){10}", "((?!json).)*$", r"(\w+\d+)+C"]) + def testUnsafeMatch(self, pattern): + with pytest.raises(SafeRe.UnsafePatternError) as err: + SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!") + assert "Potentially unsafe" in str(err.value) + + @pytest.mark.parametrize("pattern", ["^(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)(.*a)$"]) + def testUnsafeRepetition(self, pattern): + with pytest.raises(SafeRe.UnsafePatternError) as err: + SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!") + assert "More than" in str(err.value) diff --git a/src/Test/TestSite.py b/src/Test/TestSite.py new file mode 100644 index 00000000..05bb2ed9 --- /dev/null +++ b/src/Test/TestSite.py @@ -0,0 +1,70 @@ +import shutil +import os + +import pytest +from Site import SiteManager + +TEST_DATA_PATH = "src/Test/testdata" + +@pytest.mark.usefixtures("resetSettings") +class TestSite: + def testClone(self, site): + assert site.storage.directory == TEST_DATA_PATH + "/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" + + # Remove old files + if os.path.isdir(TEST_DATA_PATH + "/159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL"): + shutil.rmtree(TEST_DATA_PATH + "/159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL") + assert not os.path.isfile(TEST_DATA_PATH + "/159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL/content.json") + + # Clone 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT to 15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc + new_site = site.clone( + "159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL", "5JU2p5h3R7B1WrbaEdEDNZR7YHqRLGcjNcqwqVQzX2H4SuNe2ee", address_index=1 + ) + + # Check if clone was successful + assert new_site.address == "159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL" + assert new_site.storage.isFile("content.json") + assert new_site.storage.isFile("index.html") + assert new_site.storage.isFile("data/users/content.json") + assert new_site.storage.isFile("data/zeroblog.db") + assert new_site.storage.verifyFiles()["bad_files"] == [] # No bad files allowed + assert new_site.storage.query("SELECT * FROM keyvalue WHERE key = 'title'").fetchone()["value"] == "MyZeroBlog" + + # Optional files should be removed + + assert len(new_site.storage.loadJson("content.json").get("files_optional", {})) == 0 + + # Test re-cloning (updating) + + # Changes in non-data files should be overwritten + new_site.storage.write("index.html", b"this will be overwritten") + assert new_site.storage.read("index.html") == b"this will be overwritten" + + # Changes in data file should be kept after re-cloning + changed_contentjson = new_site.storage.loadJson("content.json") + changed_contentjson["description"] = "Update Description Test" + new_site.storage.writeJson("content.json", changed_contentjson) + + changed_data = new_site.storage.loadJson("data/data.json") + changed_data["title"] = "UpdateTest" + new_site.storage.writeJson("data/data.json", changed_data) + + # The update should be reflected to database + assert new_site.storage.query("SELECT * FROM keyvalue WHERE key = 'title'").fetchone()["value"] == "UpdateTest" + + # Re-clone the site + site.log.debug("Re-cloning") + site.clone("159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL") + + assert new_site.storage.loadJson("data/data.json")["title"] == "UpdateTest" + assert new_site.storage.loadJson("content.json")["description"] == "Update Description Test" + assert new_site.storage.read("index.html") != "this will be overwritten" + + # Delete created files + new_site.storage.deleteFiles() + assert not os.path.isdir(TEST_DATA_PATH + "/159EGD5srUsMP97UpcLy8AtKQbQLK2AbbL") + + # Delete from site registry + assert new_site.address in SiteManager.site_manager.sites + SiteManager.site_manager.delete(new_site.address) + assert new_site.address not in SiteManager.site_manager.sites diff --git a/src/Test/TestSiteDownload.py b/src/Test/TestSiteDownload.py new file mode 100644 index 00000000..cd0a4c9f --- /dev/null +++ b/src/Test/TestSiteDownload.py @@ -0,0 +1,562 @@ +import time + +import pytest +import mock +import gevent +import gevent.event +import os + +from Connection import ConnectionServer +from Config import config +from File import FileRequest +from File import FileServer +from Site.Site import Site +from . import Spy + + +@pytest.mark.usefixtures("resetTempSettings") +@pytest.mark.usefixtures("resetSettings") +class TestSiteDownload: + def testRename(self, file_server, site, site_temp): + assert site.storage.directory == config.data_dir + "/" + site.address + assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + + site_temp.addPeer(file_server.ip, 1544) + + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + + assert site_temp.storage.isFile("content.json") + + # Rename non-optional file + os.rename(site.storage.getPath("data/img/domain.png"), site.storage.getPath("data/img/domain-new.png")) + + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + + content = site.storage.loadJson("content.json") + assert "data/img/domain-new.png" in content["files"] + assert "data/img/domain.png" not in content["files"] + assert not site_temp.storage.isFile("data/img/domain-new.png") + assert site_temp.storage.isFile("data/img/domain.png") + settings_before = site_temp.settings + + with Spy.Spy(FileRequest, "route") as requests: + site.publish() + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) # Wait for download + assert "streamFile" not in [req[1] for req in requests] + + content = site_temp.storage.loadJson("content.json") + assert "data/img/domain-new.png" in content["files"] + assert "data/img/domain.png" not in content["files"] + assert site_temp.storage.isFile("data/img/domain-new.png") + assert not site_temp.storage.isFile("data/img/domain.png") + + assert site_temp.settings["size"] == settings_before["size"] + assert site_temp.settings["size_optional"] == settings_before["size_optional"] + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] + + def testRenameOptional(self, file_server, site, site_temp): + assert site.storage.directory == config.data_dir + "/" + site.address + assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + + site_temp.addPeer(file_server.ip, 1544) + + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + + assert site_temp.settings["optional_downloaded"] == 0 + + site_temp.needFile("data/optional.txt") + + assert site_temp.settings["optional_downloaded"] > 0 + settings_before = site_temp.settings + hashfield_before = site_temp.content_manager.hashfield.tobytes() + + # Rename optional file + os.rename(site.storage.getPath("data/optional.txt"), site.storage.getPath("data/optional-new.txt")) + + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", remove_missing_optional=True) + + content = site.storage.loadJson("content.json") + assert "data/optional-new.txt" in content["files_optional"] + assert "data/optional.txt" not in content["files_optional"] + assert not site_temp.storage.isFile("data/optional-new.txt") + assert site_temp.storage.isFile("data/optional.txt") + + with Spy.Spy(FileRequest, "route") as requests: + site.publish() + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) # Wait for download + assert "streamFile" not in [req[1] for req in requests] + + content = site_temp.storage.loadJson("content.json") + assert "data/optional-new.txt" in content["files_optional"] + assert "data/optional.txt" not in content["files_optional"] + assert site_temp.storage.isFile("data/optional-new.txt") + assert not site_temp.storage.isFile("data/optional.txt") + + assert site_temp.settings["size"] == settings_before["size"] + assert site_temp.settings["size_optional"] == settings_before["size_optional"] + assert site_temp.settings["optional_downloaded"] == settings_before["optional_downloaded"] + assert site_temp.content_manager.hashfield.tobytes() == hashfield_before + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] + + + def testArchivedDownload(self, file_server, site, site_temp): + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Download normally + site_temp.addPeer(file_server.ip, 1544) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"] + + assert not bad_files + assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" in site_temp.content_manager.contents + assert site_temp.storage.isFile("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json") + assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 2 + + # Add archived data + assert "archived" not in site.content_manager.contents["data/users/content.json"]["user_contents"] + assert not site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", time.time()-1) + + site.content_manager.contents["data/users/content.json"]["user_contents"]["archived"] = {"1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q": time.time()} + site.content_manager.sign("data/users/content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + + date_archived = site.content_manager.contents["data/users/content.json"]["user_contents"]["archived"]["1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q"] + assert site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived-1) + assert site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived) + assert not site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived+1) # Allow user to update archived data later + + # Push archived update + assert not "archived" in site_temp.content_manager.contents["data/users/content.json"]["user_contents"] + site.publish() + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) # Wait for download + + # The archived content should disappear from remote client + assert "archived" in site_temp.content_manager.contents["data/users/content.json"]["user_contents"] + assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" not in site_temp.content_manager.contents + assert not site_temp.storage.isDir("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q") + assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 1 + assert len(list(site_temp.storage.query("SELECT * FROM json WHERE directory LIKE '%1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q%'"))) == 0 + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] + + def testArchivedBeforeDownload(self, file_server, site, site_temp): + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Download normally + site_temp.addPeer(file_server.ip, 1544) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"] + + assert not bad_files + assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" in site_temp.content_manager.contents + assert site_temp.storage.isFile("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json") + assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 2 + + # Add archived data + assert not "archived_before" in site.content_manager.contents["data/users/content.json"]["user_contents"] + assert not site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", time.time()-1) + + content_modification_time = site.content_manager.contents["data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json"]["modified"] + site.content_manager.contents["data/users/content.json"]["user_contents"]["archived_before"] = content_modification_time + site.content_manager.sign("data/users/content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + + date_archived = site.content_manager.contents["data/users/content.json"]["user_contents"]["archived_before"] + assert site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived-1) + assert site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived) + assert not site.content_manager.isArchived("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived+1) # Allow user to update archived data later + + # Push archived update + assert not "archived_before" in site_temp.content_manager.contents["data/users/content.json"]["user_contents"] + site.publish() + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) # Wait for download + + # The archived content should disappear from remote client + assert "archived_before" in site_temp.content_manager.contents["data/users/content.json"]["user_contents"] + assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" not in site_temp.content_manager.contents + assert not site_temp.storage.isDir("data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q") + assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 1 + assert len(list(site_temp.storage.query("SELECT * FROM json WHERE directory LIKE '%1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q%'"))) == 0 + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] + + + # Test when connected peer has the optional file + def testOptionalDownload(self, file_server, site, site_temp): + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = ConnectionServer(file_server.ip, 1545) + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + site_temp.addPeer(file_server.ip, 1544) + + # Download site + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + + # Download optional data/optional.txt + site.storage.verifyFiles(quick_check=True) # Find what optional files we have + optional_file_info = site_temp.content_manager.getFileInfo("data/optional.txt") + assert site.content_manager.hashfield.hasHash(optional_file_info["sha512"]) + assert not site_temp.content_manager.hashfield.hasHash(optional_file_info["sha512"]) + + assert not site_temp.storage.isFile("data/optional.txt") + assert site.storage.isFile("data/optional.txt") + site_temp.needFile("data/optional.txt") + assert site_temp.storage.isFile("data/optional.txt") + + # Optional user file + assert not site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + optional_file_info = site_temp.content_manager.getFileInfo( + "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif" + ) + assert site.content_manager.hashfield.hasHash(optional_file_info["sha512"]) + assert not site_temp.content_manager.hashfield.hasHash(optional_file_info["sha512"]) + + site_temp.needFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert site_temp.content_manager.hashfield.hasHash(optional_file_info["sha512"]) + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] + + # Test when connected peer does not has the file, so ask him if he know someone who has it + def testFindOptional(self, file_server, site, site_temp): + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init full source server (has optional files) + site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") + file_server_full = FileServer(file_server.ip, 1546) + site_full.connection_server = file_server_full + + def listen(): + ConnectionServer.start(file_server_full) + ConnectionServer.listen(file_server_full) + + gevent.spawn(listen) + time.sleep(0.001) # Port opening + file_server_full.sites[site_full.address] = site_full # Add site + site_full.storage.verifyFiles(quick_check=True) # Check optional files + site_full_peer = site.addPeer(file_server.ip, 1546) # Add it to source server + hashfield = site_full_peer.updateHashfield() # Update hashfield + assert len(site_full.content_manager.hashfield) == 8 + assert hashfield + assert site_full.storage.isFile("data/optional.txt") + assert site_full.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert len(site_full_peer.hashfield) == 8 + + # Remove hashes from source server + for hash in list(site.content_manager.hashfield): + site.content_manager.hashfield.remove(hash) + + # Init client server + site_temp.connection_server = ConnectionServer(file_server.ip, 1545) + site_temp.addPeer(file_server.ip, 1544) # Add source server + + # Download normal files + site_temp.log.info("Start Downloading site") + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + + # Download optional data/optional.txt + optional_file_info = site_temp.content_manager.getFileInfo("data/optional.txt") + optional_file_info2 = site_temp.content_manager.getFileInfo("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert not site_temp.storage.isFile("data/optional.txt") + assert not site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert not site.content_manager.hashfield.hasHash(optional_file_info["sha512"]) # Source server don't know he has the file + assert not site.content_manager.hashfield.hasHash(optional_file_info2["sha512"]) # Source server don't know he has the file + assert site_full_peer.hashfield.hasHash(optional_file_info["sha512"]) # Source full peer on source server has the file + assert site_full_peer.hashfield.hasHash(optional_file_info2["sha512"]) # Source full peer on source server has the file + assert site_full.content_manager.hashfield.hasHash(optional_file_info["sha512"]) # Source full server he has the file + assert site_full.content_manager.hashfield.hasHash(optional_file_info2["sha512"]) # Source full server he has the file + + site_temp.log.info("Request optional files") + with Spy.Spy(FileRequest, "route") as requests: + # Request 2 file same time + threads = [] + threads.append(site_temp.needFile("data/optional.txt", blocking=False)) + threads.append(site_temp.needFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif", blocking=False)) + gevent.joinall(threads) + + assert len([request for request in requests if request[1] == "findHashIds"]) == 1 # findHashids should call only once + + assert site_temp.storage.isFile("data/optional.txt") + assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + + assert site_temp.storage.deleteFiles() + file_server_full.stop() + [connection.close() for connection in file_server.connections] + site_full.content_manager.contents.db.close("FindOptional test end") + + def testUpdate(self, file_server, site, site_temp): + assert site.storage.directory == config.data_dir + "/" + site.address + assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Don't try to find peers from the net + site.announce = mock.MagicMock(return_value=True) + site_temp.announce = mock.MagicMock(return_value=True) + + # Connect peers + site_temp.addPeer(file_server.ip, 1544) + + # Download site from site to site_temp + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + assert len(site_temp.bad_files) == 1 + + # Update file + data_original = site.storage.open("data/data.json").read() + data_new = data_original.replace(b'"ZeroBlog"', b'"UpdatedZeroBlog"') + assert data_original != data_new + + site.storage.open("data/data.json", "wb").write(data_new) + + assert site.storage.open("data/data.json").read() == data_new + assert site_temp.storage.open("data/data.json").read() == data_original + + site.log.info("Publish new data.json without patch") + # Publish without patch + with Spy.Spy(FileRequest, "route") as requests: + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + site.publish() + time.sleep(0.1) + site.log.info("Downloading site") + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + assert len([request for request in requests if request[1] in ("getFile", "streamFile")]) == 1 + + assert site_temp.storage.open("data/data.json").read() == data_new + + # Close connection to avoid update spam limit + list(site.peers.values())[0].remove() + site.addPeer(file_server.ip, 1545) + list(site_temp.peers.values())[0].ping() # Connect back + time.sleep(0.1) + + # Update with patch + data_new = data_original.replace(b'"ZeroBlog"', b'"PatchedZeroBlog"') + assert data_original != data_new + + site.storage.open("data/data.json-new", "wb").write(data_new) + + assert site.storage.open("data/data.json-new").read() == data_new + assert site_temp.storage.open("data/data.json").read() != data_new + + # Generate diff + diffs = site.content_manager.getDiffs("content.json") + assert not site.storage.isFile("data/data.json-new") # New data file removed + assert site.storage.open("data/data.json").read() == data_new # -new postfix removed + assert "data/data.json" in diffs + assert diffs["data/data.json"] == [('=', 2), ('-', 29), ('+', [b'\t"title": "PatchedZeroBlog",\n']), ('=', 31102)] + + # Publish with patch + site.log.info("Publish new data.json with patch") + with Spy.Spy(FileRequest, "route") as requests: + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + + event_done = gevent.event.AsyncResult() + site.publish(diffs=diffs) + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + assert [request for request in requests if request[1] in ("getFile", "streamFile")] == [] + + assert site_temp.storage.open("data/data.json").read() == data_new + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] + + def testBigUpdate(self, file_server, site, site_temp): + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Connect peers + site_temp.addPeer(file_server.ip, 1544) + + # Download site from site to site_temp + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + assert list(site_temp.bad_files.keys()) == ["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] + + # Update file + data_original = site.storage.open("data/data.json").read() + data_new = data_original.replace(b'"ZeroBlog"', b'"PatchedZeroBlog"') + assert data_original != data_new + + site.storage.open("data/data.json-new", "wb").write(data_new) + + assert site.storage.open("data/data.json-new").read() == data_new + assert site_temp.storage.open("data/data.json").read() != data_new + + # Generate diff + diffs = site.content_manager.getDiffs("content.json") + assert not site.storage.isFile("data/data.json-new") # New data file removed + assert site.storage.open("data/data.json").read() == data_new # -new postfix removed + assert "data/data.json" in diffs + + content_json = site.storage.loadJson("content.json") + content_json["description"] = "BigZeroBlog" * 1024 * 10 + site.storage.writeJson("content.json", content_json) + site.content_manager.loadContent("content.json", force=True) + + # Publish with patch + site.log.info("Publish new data.json with patch") + with Spy.Spy(FileRequest, "route") as requests: + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + assert site.storage.getSize("content.json") > 10 * 1024 # Make it a big content.json + site.publish(diffs=diffs) + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + file_requests = [request for request in requests if request[1] in ("getFile", "streamFile")] + assert len(file_requests) == 1 + + assert site_temp.storage.open("data/data.json").read() == data_new + assert site_temp.storage.open("content.json").read() == site.storage.open("content.json").read() + + # Test what happened if the content.json of the site is bigger than the site limit + def testHugeContentSiteUpdate(self, file_server, site, site_temp): + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Connect peers + site_temp.addPeer(file_server.ip, 1544) + + # Download site from site to site_temp + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + site_temp.settings["size_limit"] = int(20 * 1024 *1024) + site_temp.saveSettings() + + # Raise limit size to 20MB on site so it can be signed + site.settings["size_limit"] = int(20 * 1024 *1024) + site.saveSettings() + + content_json = site.storage.loadJson("content.json") + content_json["description"] = "PartirUnJour" * 1024 * 1024 + site.storage.writeJson("content.json", content_json) + changed, deleted = site.content_manager.loadContent("content.json", force=True) + + # Make sure we have 2 differents content.json + assert site_temp.storage.open("content.json").read() != site.storage.open("content.json").read() + + # Generate diff + diffs = site.content_manager.getDiffs("content.json") + + # Publish with patch + site.log.info("Publish new content.json bigger than 10MB") + with Spy.Spy(FileRequest, "route") as requests: + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + assert site.storage.getSize("content.json") > 10 * 1024 * 1024 # verify it over 10MB + time.sleep(0.1) + site.publish(diffs=diffs) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + + assert site_temp.storage.getSize("content.json") < site_temp.getSizeLimit() * 1024 * 1024 + assert site_temp.storage.open("content.json").read() == site.storage.open("content.json").read() + + def testUnicodeFilename(self, file_server, site, site_temp): + assert site.storage.directory == config.data_dir + "/" + site.address + assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + site_temp.addPeer(file_server.ip, 1544) + + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) + + site.storage.write("data/img/ÃĄrvíztÅąrő.png", b"test") + + site.content_manager.sign("content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") + + content = site.storage.loadJson("content.json") + assert "data/img/ÃĄrvíztÅąrő.png" in content["files"] + assert not site_temp.storage.isFile("data/img/ÃĄrvíztÅąrő.png") + settings_before = site_temp.settings + + with Spy.Spy(FileRequest, "route") as requests: + site.publish() + time.sleep(0.1) + assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) # Wait for download + assert len([req[1] for req in requests if req[1] == "streamFile"]) == 1 + + content = site_temp.storage.loadJson("content.json") + assert "data/img/ÃĄrvíztÅąrő.png" in content["files"] + assert site_temp.storage.isFile("data/img/ÃĄrvíztÅąrő.png") + + assert site_temp.settings["size"] == settings_before["size"] + assert site_temp.settings["size_optional"] == settings_before["size_optional"] + + assert site_temp.storage.deleteFiles() + [connection.close() for connection in file_server.connections] diff --git a/src/Test/TestSiteStorage.py b/src/Test/TestSiteStorage.py new file mode 100644 index 00000000..f11262bf --- /dev/null +++ b/src/Test/TestSiteStorage.py @@ -0,0 +1,25 @@ +import pytest + + +@pytest.mark.usefixtures("resetSettings") +class TestSiteStorage: + def testWalk(self, site): + # Rootdir + walk_root = list(site.storage.walk("")) + assert "content.json" in walk_root + assert "css/all.css" in walk_root + + # Subdir + assert list(site.storage.walk("data-default")) == ["data.json", "users/content-default.json"] + + def testList(self, site): + # Rootdir + list_root = list(site.storage.list("")) + assert "content.json" in list_root + assert "css/all.css" not in list_root + + # Subdir + assert set(site.storage.list("data-default")) == set(["data.json", "users"]) + + def testDbRebuild(self, site): + assert site.storage.rebuildDb() diff --git a/src/Test/TestThreadPool.py b/src/Test/TestThreadPool.py new file mode 100644 index 00000000..5e95005e --- /dev/null +++ b/src/Test/TestThreadPool.py @@ -0,0 +1,163 @@ +import time +import threading + +import gevent +import pytest + +from util import ThreadPool + + +class TestThreadPool: + def testExecutionOrder(self): + with ThreadPool.ThreadPool(4) as pool: + events = [] + + @pool.wrap + def blocker(): + events.append("S") + out = 0 + for i in range(10000000): + if i == 3000000: + events.append("M") + out += 1 + events.append("D") + return out + + threads = [] + for i in range(3): + threads.append(gevent.spawn(blocker)) + gevent.joinall(threads) + + assert events == ["S"] * 3 + ["M"] * 3 + ["D"] * 3 + + res = blocker() + assert res == 10000000 + + def testLockBlockingSameThread(self): + lock = ThreadPool.Lock() + + s = time.time() + + def unlocker(): + time.sleep(1) + lock.release() + + gevent.spawn(unlocker) + lock.acquire(True) + lock.acquire(True, timeout=2) + + unlock_taken = time.time() - s + + assert 1.0 < unlock_taken < 1.5 + + def testLockBlockingDifferentThread(self): + lock = ThreadPool.Lock() + + def locker(): + lock.acquire(True) + time.sleep(0.5) + lock.release() + + with ThreadPool.ThreadPool(10) as pool: + threads = [ + pool.spawn(locker), + pool.spawn(locker), + gevent.spawn(locker), + pool.spawn(locker) + ] + time.sleep(0.1) + + s = time.time() + + lock.acquire(True, 5.0) + + unlock_taken = time.time() - s + + assert 1.8 < unlock_taken < 2.2 + + gevent.joinall(threads) + + def testMainLoopCallerThreadId(self): + main_thread_id = threading.current_thread().ident + with ThreadPool.ThreadPool(5) as pool: + def getThreadId(*args, **kwargs): + return threading.current_thread().ident + + t = pool.spawn(getThreadId) + assert t.get() != main_thread_id + + t = pool.spawn(lambda: ThreadPool.main_loop.call(getThreadId)) + assert t.get() == main_thread_id + + def testMainLoopCallerGeventSpawn(self): + main_thread_id = threading.current_thread().ident + with ThreadPool.ThreadPool(5) as pool: + def waiter(): + time.sleep(1) + return threading.current_thread().ident + + def geventSpawner(): + event = ThreadPool.main_loop.call(gevent.spawn, waiter) + + with pytest.raises(Exception) as greenlet_err: + event.get() + assert str(greenlet_err.value) == "cannot switch to a different thread" + + waiter_thread_id = ThreadPool.main_loop.call(event.get) + return waiter_thread_id + + s = time.time() + waiter_thread_id = pool.apply(geventSpawner) + assert main_thread_id == waiter_thread_id + time_taken = time.time() - s + assert 0.9 < time_taken < 1.2 + + def testEvent(self): + with ThreadPool.ThreadPool(5) as pool: + event = ThreadPool.Event() + + def setter(): + time.sleep(1) + event.set("done!") + + def getter(): + return event.get() + + pool.spawn(setter) + t_gevent = gevent.spawn(getter) + t_pool = pool.spawn(getter) + s = time.time() + assert event.get() == "done!" + time_taken = time.time() - s + gevent.joinall([t_gevent, t_pool]) + + assert t_gevent.get() == "done!" + assert t_pool.get() == "done!" + + assert 0.9 < time_taken < 1.2 + + with pytest.raises(Exception) as err: + event.set("another result") + + assert "Event already has value" in str(err.value) + + def testMemoryLeak(self): + import gc + thread_objs_before = [id(obj) for obj in gc.get_objects() if "threadpool" in str(type(obj))] + + def worker(): + time.sleep(0.1) + return "ok" + + def poolTest(): + with ThreadPool.ThreadPool(5) as pool: + for i in range(20): + pool.spawn(worker) + + for i in range(5): + poolTest() + new_thread_objs = [obj for obj in gc.get_objects() if "threadpool" in str(type(obj)) and id(obj) not in thread_objs_before] + #print("New objs:", new_thread_objs, "run:", num_run) + + # Make sure no threadpool object left behind + assert not new_thread_objs diff --git a/src/Test/TestTor.py b/src/Test/TestTor.py new file mode 100644 index 00000000..e6b82c1a --- /dev/null +++ b/src/Test/TestTor.py @@ -0,0 +1,153 @@ +import time + +import pytest +import mock + +from File import FileServer +from Crypt import CryptTor +from Config import config + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestTor: + def testDownload(self, tor_manager): + for retry in range(15): + time.sleep(1) + if tor_manager.enabled and tor_manager.conn: + break + assert tor_manager.enabled + + def testManagerConnection(self, tor_manager): + assert "250-version" in tor_manager.request("GETINFO version") + + def testAddOnion(self, tor_manager): + # Add + address = tor_manager.addOnion() + assert address + assert address in tor_manager.privatekeys + + # Delete + assert tor_manager.delOnion(address) + assert address not in tor_manager.privatekeys + + def testSignOnion(self, tor_manager): + address = tor_manager.addOnion() + + # Sign + sign = CryptTor.sign(b"hello", tor_manager.getPrivatekey(address)) + assert len(sign) == 128 + + # Verify + publickey = CryptTor.privatekeyToPublickey(tor_manager.getPrivatekey(address)) + assert len(publickey) == 140 + assert CryptTor.verify(b"hello", publickey, sign) + assert not CryptTor.verify(b"not hello", publickey, sign) + + # Pub to address + assert CryptTor.publickeyToOnion(publickey) == address + + # Delete + tor_manager.delOnion(address) + + @pytest.mark.slow + def testConnection(self, tor_manager, file_server, site, site_temp): + file_server.tor_manager.start_onions = True + address = file_server.tor_manager.getOnion(site.address) + assert address + print("Connecting to", address) + for retry in range(5): # Wait for hidden service creation + time.sleep(10) + try: + connection = file_server.getConnection(address + ".onion", 1544) + if connection: + break + except Exception as err: + continue + assert connection.handshake + assert not connection.handshake["peer_id"] # No peer_id for Tor connections + + # Return the same connection without site specified + assert file_server.getConnection(address + ".onion", 1544) == connection + # No reuse for different site + assert file_server.getConnection(address + ".onion", 1544, site=site) != connection + assert file_server.getConnection(address + ".onion", 1544, site=site) == file_server.getConnection(address + ".onion", 1544, site=site) + site_temp.address = "1OTHERSITE" + assert file_server.getConnection(address + ".onion", 1544, site=site) != file_server.getConnection(address + ".onion", 1544, site=site_temp) + + # Only allow to query from the locked site + file_server.sites[site.address] = site + connection_locked = file_server.getConnection(address + ".onion", 1544, site=site) + assert "body" in connection_locked.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0}) + assert connection_locked.request("getFile", {"site": "1OTHERSITE", "inner_path": "content.json", "location": 0})["error"] == "Invalid site" + + def testPex(self, file_server, site, site_temp): + # Register site to currently running fileserver + site.connection_server = file_server + file_server.sites[site.address] = site + # Create a new file server to emulate new peer connecting to our peer + file_server_temp = FileServer(file_server.ip, 1545) + site_temp.connection_server = file_server_temp + file_server_temp.sites[site_temp.address] = site_temp + + # We will request peers from this + peer_source = site_temp.addPeer(file_server.ip, 1544) + + # Get ip4 peers from source site + site.addPeer("1.2.3.4", 1555) # Add peer to source site + assert peer_source.pex(need_num=10) == 1 + assert len(site_temp.peers) == 2 + assert "1.2.3.4:1555" in site_temp.peers + + # Get onion peers from source site + site.addPeer("bka4ht2bzxchy44r.onion", 1555) + assert "bka4ht2bzxchy44r.onion:1555" not in site_temp.peers + + # Don't add onion peers if not supported + assert "onion" not in file_server_temp.supported_ip_types + assert peer_source.pex(need_num=10) == 0 + + file_server_temp.supported_ip_types.append("onion") + assert peer_source.pex(need_num=10) == 1 + + assert "bka4ht2bzxchy44r.onion:1555" in site_temp.peers + + def testFindHash(self, tor_manager, file_server, site, site_temp): + file_server.ip_incoming = {} # Reset flood protection + file_server.sites[site.address] = site + file_server.tor_manager = tor_manager + + client = FileServer(file_server.ip, 1545) + client.sites = {site_temp.address: site_temp} + site_temp.connection_server = client + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer(file_server.ip, 1544) + + assert peer_file_server.findHashIds([1234]) == {} + + # Add fake peer with requred hash + fake_peer_1 = site.addPeer("bka4ht2bzxchy44r.onion", 1544) + fake_peer_1.hashfield.append(1234) + fake_peer_2 = site.addPeer("1.2.3.5", 1545) + fake_peer_2.hashfield.append(1234) + fake_peer_2.hashfield.append(1235) + fake_peer_3 = site.addPeer("1.2.3.6", 1546) + fake_peer_3.hashfield.append(1235) + fake_peer_3.hashfield.append(1236) + + res = peer_file_server.findHashIds([1234, 1235]) + + assert sorted(res[1234]) == [('1.2.3.5', 1545), ("bka4ht2bzxchy44r.onion", 1544)] + assert sorted(res[1235]) == [('1.2.3.5', 1545), ('1.2.3.6', 1546)] + + # Test my address adding + site.content_manager.hashfield.append(1234) + + res = peer_file_server.findHashIds([1234, 1235]) + assert sorted(res[1234]) == [('1.2.3.5', 1545), (file_server.ip, 1544), ("bka4ht2bzxchy44r.onion", 1544)] + assert sorted(res[1235]) == [('1.2.3.5', 1545), ('1.2.3.6', 1546)] + + def testSiteOnion(self, tor_manager): + with mock.patch.object(config, "tor", "always"): + assert tor_manager.getOnion("address1") != tor_manager.getOnion("address2") + assert tor_manager.getOnion("address1") == tor_manager.getOnion("address1") diff --git a/src/Test/TestTranslate.py b/src/Test/TestTranslate.py new file mode 100644 index 00000000..348a65a6 --- /dev/null +++ b/src/Test/TestTranslate.py @@ -0,0 +1,61 @@ +from Translate import Translate + +class TestTranslate: + def testTranslateStrict(self): + translate = Translate() + data = """ + translated = _("original") + not_translated = "original" + """ + data_translated = translate.translateData(data, {"_(original)": "translated"}) + assert 'translated = _("translated")' in data_translated + assert 'not_translated = "original"' in data_translated + + def testTranslateStrictNamed(self): + translate = Translate() + data = """ + translated = _("original", "original named") + translated_other = _("original", "original other named") + not_translated = "original" + """ + data_translated = translate.translateData(data, {"_(original, original named)": "translated"}) + assert 'translated = _("translated")' in data_translated + assert 'not_translated = "original"' in data_translated + + def testTranslateUtf8(self): + translate = Translate() + data = """ + greeting = "Hi again ÃĄrvztÅąrőtÃļkÃļrfÃērÃŗgÊp!" + """ + data_translated = translate.translateData(data, {"Hi again ÃĄrvztÅąrőtÃļkÃļrfÃērÃŗgÊp!": "Üdv Ãējra ÃĄrvztÅąrőtÃļkÃļrfÃērÃŗgÊp!"}) + assert data_translated == """ + greeting = "Üdv Ãējra ÃĄrvztÅąrőtÃļkÃļrfÃērÃŗgÊp!" + """ + + def testTranslateEscape(self): + _ = Translate() + _["Hello"] = "Szia" + + # Simple escaping + data = "{_[Hello]} {username}!" + username = "Hacker" + data_translated = _(data) + assert 'Szia' in data_translated + assert '<' not in data_translated + assert data_translated == "Szia Hacker<script>alert('boom')</script>!" + + # Escaping dicts + user = {"username": "Hacker"} + data = "{_[Hello]} {user[username]}!" + data_translated = _(data) + assert 'Szia' in data_translated + assert '<' not in data_translated + assert data_translated == "Szia Hacker<script>alert('boom')</script>!" + + # Escaping lists + users = [{"username": "Hacker"}] + data = "{_[Hello]} {users[0][username]}!" + data_translated = _(data) + assert 'Szia' in data_translated + assert '<' not in data_translated + assert data_translated == "Szia Hacker<script>alert('boom')</script>!" diff --git a/src/Test/TestUiWebsocket.py b/src/Test/TestUiWebsocket.py new file mode 100644 index 00000000..d2d23d03 --- /dev/null +++ b/src/Test/TestUiWebsocket.py @@ -0,0 +1,11 @@ +import sys +import pytest + +@pytest.mark.usefixtures("resetSettings") +class TestUiWebsocket: + def testPermission(self, ui_websocket): + res = ui_websocket.testAction("ping") + assert res == "pong" + + res = ui_websocket.testAction("certList") + assert "You don't have permission" in res["error"] diff --git a/src/Test/TestUpnpPunch.py b/src/Test/TestUpnpPunch.py new file mode 100644 index 00000000..f17c77bd --- /dev/null +++ b/src/Test/TestUpnpPunch.py @@ -0,0 +1,274 @@ +import socket +from urllib.parse import urlparse + +import pytest +import mock + +from util import UpnpPunch as upnp + + +@pytest.fixture +def mock_socket(): + mock_socket = mock.MagicMock() + mock_socket.recv = mock.MagicMock(return_value=b'Hello') + mock_socket.bind = mock.MagicMock() + mock_socket.send_to = mock.MagicMock() + + return mock_socket + + +@pytest.fixture +def url_obj(): + return urlparse('http://192.168.1.1/ctrlPoint.xml') + + +@pytest.fixture(params=['WANPPPConnection', 'WANIPConnection']) +def igd_profile(request): + return """ + urn:schemas-upnp-org:service:{}:1 + urn:upnp-org:serviceId:wanpppc:pppoa + /upnp/control/wanpppcpppoa + /upnp/event/wanpppcpppoa + /WANPPPConnection.xml +""".format(request.param) + + +@pytest.fixture +def httplib_response(): + class FakeResponse(object): + def __init__(self, status=200, body='OK'): + self.status = status + self.body = body + + def read(self): + return self.body + return FakeResponse + + +class TestUpnpPunch(object): + def test_perform_m_search(self, mock_socket): + local_ip = '127.0.0.1' + + with mock.patch('util.UpnpPunch.socket.socket', + return_value=mock_socket): + result = upnp.perform_m_search(local_ip) + assert result == 'Hello' + assert local_ip == mock_socket.bind.call_args_list[0][0][0][0] + assert ('239.255.255.250', + 1900) == mock_socket.sendto.call_args_list[0][0][1] + + def test_perform_m_search_socket_error(self, mock_socket): + mock_socket.recv.side_effect = socket.error('Timeout error') + + with mock.patch('util.UpnpPunch.socket.socket', + return_value=mock_socket): + with pytest.raises(upnp.UpnpError): + upnp.perform_m_search('127.0.0.1') + + def test_retrieve_location_from_ssdp(self, url_obj): + ctrl_location = url_obj.geturl() + parsed_location = urlparse(ctrl_location) + rsp = ('auth: gibberish\r\nlocation: {0}\r\n' + 'Content-Type: text/html\r\n\r\n').format(ctrl_location) + result = upnp._retrieve_location_from_ssdp(rsp) + assert result == parsed_location + + def test_retrieve_location_from_ssdp_no_header(self): + rsp = 'auth: gibberish\r\nContent-Type: application/json\r\n\r\n' + with pytest.raises(upnp.IGDError): + upnp._retrieve_location_from_ssdp(rsp) + + def test_retrieve_igd_profile(self, url_obj): + with mock.patch('urllib.request.urlopen') as mock_urlopen: + upnp._retrieve_igd_profile(url_obj) + mock_urlopen.assert_called_with(url_obj.geturl(), timeout=5) + + def test_retrieve_igd_profile_timeout(self, url_obj): + with mock.patch('urllib.request.urlopen') as mock_urlopen: + mock_urlopen.side_effect = socket.error('Timeout error') + with pytest.raises(upnp.IGDError): + upnp._retrieve_igd_profile(url_obj) + + def test_parse_igd_profile_service_type(self, igd_profile): + control_path, upnp_schema = upnp._parse_igd_profile(igd_profile) + assert control_path == '/upnp/control/wanpppcpppoa' + assert upnp_schema in ('WANPPPConnection', 'WANIPConnection',) + + def test_parse_igd_profile_no_ctrlurl(self, igd_profile): + igd_profile = igd_profile.replace('controlURL', 'nope') + with pytest.raises(upnp.IGDError): + control_path, upnp_schema = upnp._parse_igd_profile(igd_profile) + + def test_parse_igd_profile_no_schema(self, igd_profile): + igd_profile = igd_profile.replace('Connection', 'nope') + with pytest.raises(upnp.IGDError): + control_path, upnp_schema = upnp._parse_igd_profile(igd_profile) + + def test_create_open_message_parsable(self): + from xml.parsers.expat import ExpatError + msg, _ = upnp._create_open_message('127.0.0.1', 8888) + try: + upnp.parseString(msg) + except ExpatError as e: + pytest.fail('Incorrect XML message: {}'.format(e)) + + def test_create_open_message_contains_right_stuff(self): + settings = {'description': 'test desc', + 'protocol': 'test proto', + 'upnp_schema': 'test schema'} + msg, fn_name = upnp._create_open_message('127.0.0.1', 8888, **settings) + assert fn_name == 'AddPortMapping' + assert '127.0.0.1' in msg + assert '8888' in msg + assert settings['description'] in msg + assert settings['protocol'] in msg + assert settings['upnp_schema'] in msg + + def test_parse_for_errors_bad_rsp(self, httplib_response): + rsp = httplib_response(status=500) + with pytest.raises(upnp.IGDError) as err: + upnp._parse_for_errors(rsp) + assert 'Unable to parse' in str(err.value) + + def test_parse_for_errors_error(self, httplib_response): + soap_error = ('' + '500' + 'Bad request' + '') + rsp = httplib_response(status=500, body=soap_error) + with pytest.raises(upnp.IGDError) as err: + upnp._parse_for_errors(rsp) + assert 'SOAP request error' in str(err.value) + + def test_parse_for_errors_good_rsp(self, httplib_response): + rsp = httplib_response(status=200) + assert rsp == upnp._parse_for_errors(rsp) + + def test_send_requests_success(self): + with mock.patch( + 'util.UpnpPunch._send_soap_request') as mock_send_request: + mock_send_request.return_value = mock.MagicMock(status=200) + upnp._send_requests(['msg'], None, None, None) + + assert mock_send_request.called + + def test_send_requests_failed(self): + with mock.patch( + 'util.UpnpPunch._send_soap_request') as mock_send_request: + mock_send_request.return_value = mock.MagicMock(status=500) + with pytest.raises(upnp.UpnpError): + upnp._send_requests(['msg'], None, None, None) + + assert mock_send_request.called + + def test_collect_idg_data(self): + pass + + @mock.patch('util.UpnpPunch._get_local_ips') + @mock.patch('util.UpnpPunch._collect_idg_data') + @mock.patch('util.UpnpPunch._send_requests') + def test_ask_to_open_port_success(self, mock_send_requests, + mock_collect_idg, mock_local_ips): + mock_collect_idg.return_value = {'upnp_schema': 'schema-yo'} + mock_local_ips.return_value = ['192.168.0.12'] + + result = upnp.ask_to_open_port(retries=5) + + soap_msg = mock_send_requests.call_args[0][0][0][0] + + assert result is True + + assert mock_collect_idg.called + assert '192.168.0.12' in soap_msg + assert '15441' in soap_msg + assert 'schema-yo' in soap_msg + + @mock.patch('util.UpnpPunch._get_local_ips') + @mock.patch('util.UpnpPunch._collect_idg_data') + @mock.patch('util.UpnpPunch._send_requests') + def test_ask_to_open_port_failure(self, mock_send_requests, + mock_collect_idg, mock_local_ips): + mock_local_ips.return_value = ['192.168.0.12'] + mock_collect_idg.return_value = {'upnp_schema': 'schema-yo'} + mock_send_requests.side_effect = upnp.UpnpError() + + with pytest.raises(upnp.UpnpError): + upnp.ask_to_open_port() + + @mock.patch('util.UpnpPunch._collect_idg_data') + @mock.patch('util.UpnpPunch._send_requests') + def test_orchestrate_soap_request(self, mock_send_requests, + mock_collect_idg): + soap_mock = mock.MagicMock() + args = ['127.0.0.1', 31337, soap_mock, 'upnp-test', {'upnp_schema': + 'schema-yo'}] + mock_collect_idg.return_value = args[-1] + + upnp._orchestrate_soap_request(*args[:-1]) + + assert mock_collect_idg.called + soap_mock.assert_called_with( + *args[:2] + ['upnp-test', 'UDP', 'schema-yo']) + assert mock_send_requests.called + + @mock.patch('util.UpnpPunch._collect_idg_data') + @mock.patch('util.UpnpPunch._send_requests') + def test_orchestrate_soap_request_without_desc(self, mock_send_requests, + mock_collect_idg): + soap_mock = mock.MagicMock() + args = ['127.0.0.1', 31337, soap_mock, {'upnp_schema': 'schema-yo'}] + mock_collect_idg.return_value = args[-1] + + upnp._orchestrate_soap_request(*args[:-1]) + + assert mock_collect_idg.called + soap_mock.assert_called_with(*args[:2] + [None, 'UDP', 'schema-yo']) + assert mock_send_requests.called + + def test_create_close_message_parsable(self): + from xml.parsers.expat import ExpatError + msg, _ = upnp._create_close_message('127.0.0.1', 8888) + try: + upnp.parseString(msg) + except ExpatError as e: + pytest.fail('Incorrect XML message: {}'.format(e)) + + def test_create_close_message_contains_right_stuff(self): + settings = {'protocol': 'test proto', + 'upnp_schema': 'test schema'} + msg, fn_name = upnp._create_close_message('127.0.0.1', 8888, ** + settings) + assert fn_name == 'DeletePortMapping' + assert '8888' in msg + assert settings['protocol'] in msg + assert settings['upnp_schema'] in msg + + @mock.patch('util.UpnpPunch._get_local_ips') + @mock.patch('util.UpnpPunch._orchestrate_soap_request') + def test_communicate_with_igd_success(self, mock_orchestrate, + mock_get_local_ips): + mock_get_local_ips.return_value = ['192.168.0.12'] + upnp._communicate_with_igd() + assert mock_get_local_ips.called + assert mock_orchestrate.called + + @mock.patch('util.UpnpPunch._get_local_ips') + @mock.patch('util.UpnpPunch._orchestrate_soap_request') + def test_communicate_with_igd_succeed_despite_single_failure( + self, mock_orchestrate, mock_get_local_ips): + mock_get_local_ips.return_value = ['192.168.0.12'] + mock_orchestrate.side_effect = [upnp.UpnpError, None] + upnp._communicate_with_igd(retries=2) + assert mock_get_local_ips.called + assert mock_orchestrate.called + + @mock.patch('util.UpnpPunch._get_local_ips') + @mock.patch('util.UpnpPunch._orchestrate_soap_request') + def test_communicate_with_igd_total_failure(self, mock_orchestrate, + mock_get_local_ips): + mock_get_local_ips.return_value = ['192.168.0.12'] + mock_orchestrate.side_effect = [upnp.UpnpError, upnp.IGDError] + with pytest.raises(upnp.UpnpError): + upnp._communicate_with_igd(retries=2) + assert mock_get_local_ips.called + assert mock_orchestrate.called diff --git a/src/Test/TestUser.py b/src/Test/TestUser.py new file mode 100644 index 00000000..e5ec5c8c --- /dev/null +++ b/src/Test/TestUser.py @@ -0,0 +1,50 @@ +import pytest + +from Crypt import CryptBitcoin + + +@pytest.mark.usefixtures("resetSettings") +class TestUser: + def testAddress(self, user): + assert user.master_address == "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc" + address_index = 1458664252141532163166741013621928587528255888800826689784628722366466547364755811 + assert user.getAddressAuthIndex("15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc") == address_index + + # Re-generate privatekey based on address_index + def testNewSite(self, user): + address, address_index, site_data = user.getNewSiteData() # Create a new random site + assert CryptBitcoin.hdPrivatekey(user.master_seed, address_index) == site_data["privatekey"] + + user.sites = {} # Reset user data + + # Site address and auth address is different + assert user.getSiteData(address)["auth_address"] != address + # Re-generate auth_privatekey for site + assert user.getSiteData(address)["auth_privatekey"] == site_data["auth_privatekey"] + + def testAuthAddress(self, user): + # Auth address without Cert + auth_address = user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") + assert auth_address == "1MyJgYQjeEkR9QD66nkfJc9zqi9uUy5Lr2" + auth_privatekey = user.getAuthPrivatekey("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") + assert CryptBitcoin.privatekeyToAddress(auth_privatekey) == auth_address + + def testCert(self, user): + cert_auth_address = user.getAuthAddress("1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz") # Add site to user's registry + # Add cert + user.addCert(cert_auth_address, "zeroid.bit", "faketype", "fakeuser", "fakesign") + user.setCert("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr", "zeroid.bit") + + # By using certificate the auth address should be same as the certificate provider + assert user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == cert_auth_address + auth_privatekey = user.getAuthPrivatekey("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") + assert CryptBitcoin.privatekeyToAddress(auth_privatekey) == cert_auth_address + + # Test delete site data + assert "1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr" in user.sites + user.deleteSiteData("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") + assert "1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr" not in user.sites + + # Re-create add site should generate normal, unique auth_address + assert not user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == cert_auth_address + assert user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == "1MyJgYQjeEkR9QD66nkfJc9zqi9uUy5Lr2" diff --git a/src/Test/TestWeb.py b/src/Test/TestWeb.py new file mode 100644 index 00000000..2ce66c98 --- /dev/null +++ b/src/Test/TestWeb.py @@ -0,0 +1,105 @@ +import urllib.request + +import pytest + +try: + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support.expected_conditions import staleness_of, title_is + from selenium.common.exceptions import NoSuchElementException +except: + pass + + +class WaitForPageLoad(object): + def __init__(self, browser): + self.browser = browser + + def __enter__(self): + self.old_page = self.browser.find_element_by_tag_name('html') + + def __exit__(self, *args): + WebDriverWait(self.browser, 10).until(staleness_of(self.old_page)) + + +def getContextUrl(browser): + return browser.execute_script("return window.location.toString()") + + +def getUrl(url): + content = urllib.request.urlopen(url).read() + assert "server error" not in content.lower(), "Got a server error! " + repr(url) + return content + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.webtest +class TestWeb: + def testFileSecurity(self, site_url): + assert "Not Found" in getUrl("%s/media/sites.json" % site_url) + assert "Forbidden" in getUrl("%s/media/./sites.json" % site_url) + assert "Forbidden" in getUrl("%s/media/../config.py" % site_url) + assert "Forbidden" in getUrl("%s/media/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/../sites.json" % site_url) + assert "Forbidden" in getUrl("%s/media/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/..//sites.json" % site_url) + assert "Forbidden" in getUrl("%s/media/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/../../zeronet.py" % site_url) + + assert "Not Found" in getUrl("%s/raw/sites.json" % site_url) + assert "Forbidden" in getUrl("%s/raw/./sites.json" % site_url) + assert "Forbidden" in getUrl("%s/raw/../config.py" % site_url) + assert "Forbidden" in getUrl("%s/raw/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/../sites.json" % site_url) + assert "Forbidden" in getUrl("%s/raw/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/..//sites.json" % site_url) + assert "Forbidden" in getUrl("%s/raw/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/../../zeronet.py" % site_url) + + assert "Forbidden" in getUrl("%s/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/../sites.json" % site_url) + assert "Forbidden" in getUrl("%s/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/..//sites.json" % site_url) + assert "Forbidden" in getUrl("%s/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/../../zeronet.py" % site_url) + + assert "Forbidden" in getUrl("%s/content.db" % site_url) + assert "Forbidden" in getUrl("%s/./users.json" % site_url) + assert "Forbidden" in getUrl("%s/./key-rsa.pem" % site_url) + assert "Forbidden" in getUrl("%s/././././././././././//////sites.json" % site_url) + + def testLinkSecurity(self, browser, site_url): + browser.get("%s/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/test/security.html" % site_url) + WebDriverWait(browser, 10).until(title_is("ZeroHello - ZeroNet")) + assert getContextUrl(browser) == "%s/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/test/security.html" % site_url + + # Switch to inner frame + browser.switch_to.frame(browser.find_element_by_id("inner-iframe")) + assert "wrapper_nonce" in getContextUrl(browser) + assert browser.find_element_by_id("script_output").text == "Result: Works" + browser.switch_to.default_content() + + # Clicking on links without target + browser.switch_to.frame(browser.find_element_by_id("inner-iframe")) + with WaitForPageLoad(browser): + browser.find_element_by_id("link_to_current").click() + assert "wrapper_nonce" not in getContextUrl(browser) # The browser object back to default content + assert "Forbidden" not in browser.page_source + # Check if we have frame inside frame + browser.switch_to.frame(browser.find_element_by_id("inner-iframe")) + with pytest.raises(NoSuchElementException): + assert not browser.find_element_by_id("inner-iframe") + browser.switch_to.default_content() + + # Clicking on link with target=_top + browser.switch_to.frame(browser.find_element_by_id("inner-iframe")) + with WaitForPageLoad(browser): + browser.find_element_by_id("link_to_top").click() + assert "wrapper_nonce" not in getContextUrl(browser) # The browser object back to default content + assert "Forbidden" not in browser.page_source + browser.switch_to.default_content() + + # Try to escape from inner_frame + browser.switch_to.frame(browser.find_element_by_id("inner-iframe")) + assert "wrapper_nonce" in getContextUrl(browser) # Make sure we are inside of the inner-iframe + with WaitForPageLoad(browser): + browser.execute_script("window.top.location = window.location") + assert "wrapper_nonce" in getContextUrl(browser) # We try to use nonce-ed html without iframe + assert " 0.1: + line_marker = "!" + elif since_last > 0.02: + line_marker = "*" + elif since_last > 0.01: + line_marker = "-" + else: + line_marker = " " + + since_start = time.time() - time_start + record.since_start = "%s%.3fs" % (line_marker, since_start) + + self.time_last = time.time() + return True + +log = logging.getLogger() +fmt = logging.Formatter(fmt='%(since_start)s %(thread_marker)s %(levelname)-8s %(name)s %(message)s %(thread_title)s') +[hndl.addFilter(TimeFilter()) for hndl in log.handlers] +[hndl.setFormatter(fmt) for hndl in log.handlers] + +from Site.Site import Site +from Site import SiteManager +from User import UserManager +from File import FileServer +from Connection import ConnectionServer +from Crypt import CryptConnection +from Crypt import CryptBitcoin +from Ui import UiWebsocket +from Tor import TorManager +from Content import ContentDb +from util import RateLimit +from Db import Db +from Debug import Debug + +gevent.get_hub().NOT_ERROR += (Debug.Notify,) + +def cleanup(): + Db.dbCloseAll() + for dir_path in [config.data_dir, config.data_dir + "-temp"]: + if os.path.isdir(dir_path): + for file_name in os.listdir(dir_path): + ext = file_name.rsplit(".", 1)[-1] + if ext not in ["csr", "pem", "srl", "db", "json", "tmp"]: + continue + file_path = dir_path + "/" + file_name + if os.path.isfile(file_path): + os.unlink(file_path) + +atexit_register(cleanup) + +@pytest.fixture(scope="session") +def resetSettings(request): + open("%s/sites.json" % config.data_dir, "w").write("{}") + open("%s/filters.json" % config.data_dir, "w").write("{}") + open("%s/users.json" % config.data_dir, "w").write(""" + { + "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc": { + "certs": {}, + "master_seed": "024bceac1105483d66585d8a60eaf20aa8c3254b0f266e0d626ddb6114e2949a", + "sites": {} + } + } + """) + + +@pytest.fixture(scope="session") +def resetTempSettings(request): + data_dir_temp = config.data_dir + "-temp" + if not os.path.isdir(data_dir_temp): + os.mkdir(data_dir_temp) + open("%s/sites.json" % data_dir_temp, "w").write("{}") + open("%s/filters.json" % data_dir_temp, "w").write("{}") + open("%s/users.json" % data_dir_temp, "w").write(""" + { + "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc": { + "certs": {}, + "master_seed": "024bceac1105483d66585d8a60eaf20aa8c3254b0f266e0d626ddb6114e2949a", + "sites": {} + } + } + """) + + def cleanup(): + os.unlink("%s/sites.json" % data_dir_temp) + os.unlink("%s/users.json" % data_dir_temp) + os.unlink("%s/filters.json" % data_dir_temp) + request.addfinalizer(cleanup) + + +@pytest.fixture() +def site(request): + threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)] + # Reset ratelimit + RateLimit.queue_db = {} + RateLimit.called_db = {} + + site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") + + # Always use original data + assert "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" in site.storage.getPath("") # Make sure we dont delete everything + shutil.rmtree(site.storage.getPath(""), True) + shutil.copytree(site.storage.getPath("") + "-original", site.storage.getPath("")) + + # Add to site manager + SiteManager.site_manager.get("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") + site.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + def cleanup(): + site.delete() + site.content_manager.contents.db.close("Test cleanup") + site.content_manager.contents.db.timer_check_optional.kill() + SiteManager.site_manager.sites.clear() + db_path = "%s/content.db" % config.data_dir + os.unlink(db_path) + del ContentDb.content_dbs[db_path] + gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before]) + request.addfinalizer(cleanup) + + site.greenlet_manager.stopGreenlets() + site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") # Create new Site object to load content.json files + if not SiteManager.site_manager.sites: + SiteManager.site_manager.sites = {} + SiteManager.site_manager.sites["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] = site + site.settings["serving"] = True + return site + + +@pytest.fixture() +def site_temp(request): + threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)] + with mock.patch("Config.config.data_dir", config.data_dir + "-temp"): + site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") + site_temp.settings["serving"] = True + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + def cleanup(): + site_temp.delete() + site_temp.content_manager.contents.db.close("Test cleanup") + site_temp.content_manager.contents.db.timer_check_optional.kill() + db_path = "%s-temp/content.db" % config.data_dir + os.unlink(db_path) + del ContentDb.content_dbs[db_path] + gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before]) + request.addfinalizer(cleanup) + site_temp.log = logging.getLogger("Temp:%s" % site_temp.address_short) + return site_temp + + +@pytest.fixture(scope="session") +def user(): + user = UserManager.user_manager.get() + if not user: + user = UserManager.user_manager.create() + user.sites = {} # Reset user data + return user + + +@pytest.fixture(scope="session") +def browser(request): + try: + from selenium import webdriver + print("Starting chromedriver...") + options = webdriver.chrome.options.Options() + options.add_argument("--headless") + options.add_argument("--window-size=1920x1080") + options.add_argument("--log-level=1") + browser = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, service_log_path=os.path.devnull, options=options) + + def quit(): + browser.quit() + request.addfinalizer(quit) + except Exception as err: + raise pytest.skip("Test requires selenium + chromedriver: %s" % err) + return browser + + +@pytest.fixture(scope="session") +def site_url(): + try: + urllib.request.urlopen(SITE_URL).read() + except Exception as err: + raise pytest.skip("Test requires zeronet client running: %s" % err) + return SITE_URL + + +@pytest.fixture(params=['ipv4', 'ipv6']) +def file_server(request): + if request.param == "ipv4": + return request.getfixturevalue("file_server4") + else: + return request.getfixturevalue("file_server6") + + +@pytest.fixture +def file_server4(request): + time.sleep(0.1) + file_server = FileServer("127.0.0.1", 1544) + file_server.ip_external = "1.2.3.4" # Fake external ip + + def listen(): + ConnectionServer.start(file_server) + ConnectionServer.listen(file_server) + + gevent.spawn(listen) + # Wait for port opening + for retry in range(10): + time.sleep(0.1) # Port opening + try: + conn = file_server.getConnection("127.0.0.1", 1544) + conn.close() + break + except Exception as err: + print("FileServer6 startup error", Debug.formatException(err)) + assert file_server.running + file_server.ip_incoming = {} # Reset flood protection + + def stop(): + file_server.stop() + request.addfinalizer(stop) + return file_server + + +@pytest.fixture +def file_server6(request): + try: + sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + sock.connect(("::1", 80, 1, 1)) + has_ipv6 = True + except OSError: + has_ipv6 = False + if not has_ipv6: + pytest.skip("Ipv6 not supported") + + + time.sleep(0.1) + file_server6 = FileServer("::1", 1544) + file_server6.ip_external = 'fca5:95d6:bfde:d902:8951:276e:1111:a22c' # Fake external ip + + def listen(): + ConnectionServer.start(file_server6) + ConnectionServer.listen(file_server6) + + gevent.spawn(listen) + # Wait for port opening + for retry in range(10): + time.sleep(0.1) # Port opening + try: + conn = file_server6.getConnection("::1", 1544) + conn.close() + break + except Exception as err: + print("FileServer6 startup error", Debug.formatException(err)) + assert file_server6.running + file_server6.ip_incoming = {} # Reset flood protection + + def stop(): + file_server6.stop() + request.addfinalizer(stop) + return file_server6 + + +@pytest.fixture() +def ui_websocket(site, user): + class WsMock: + def __init__(self): + self.result = gevent.event.AsyncResult() + + def send(self, data): + logging.debug("WsMock: Set result (data: %s) called by %s" % (data, Debug.formatStack())) + self.result.set(json.loads(data)["result"]) + + def getResult(self): + logging.debug("WsMock: Get result") + back = self.result.get() + logging.debug("WsMock: Got result (data: %s)" % back) + self.result = gevent.event.AsyncResult() + return back + + ws_mock = WsMock() + ui_websocket = UiWebsocket(ws_mock, site, None, user, None) + + def testAction(action, *args, **kwargs): + ui_websocket.handleRequest({"id": 0, "cmd": action, "params": list(args) if args else kwargs}) + return ui_websocket.ws.getResult() + + ui_websocket.testAction = testAction + return ui_websocket + + +@pytest.fixture(scope="session") +def tor_manager(): + try: + tor_manager = TorManager(fileserver_port=1544) + tor_manager.start() + assert tor_manager.conn is not None + tor_manager.startOnions() + except Exception as err: + raise pytest.skip("Test requires Tor with ControlPort: %s, %s" % (config.tor_controller, err)) + return tor_manager + + +@pytest.fixture() +def db(request): + db_path = "%s/zeronet.db" % config.data_dir + schema = { + "db_name": "TestDb", + "db_file": "%s/zeronet.db" % config.data_dir, + "maps": { + "data.json": { + "to_table": [ + "test", + {"node": "test", "table": "test_importfilter", "import_cols": ["test_id", "title"]} + ] + } + }, + "tables": { + "test": { + "cols": [ + ["test_id", "INTEGER"], + ["title", "TEXT"], + ["json_id", "INTEGER REFERENCES json (json_id)"] + ], + "indexes": ["CREATE UNIQUE INDEX test_id ON test(test_id)"], + "schema_changed": 1426195822 + }, + "test_importfilter": { + "cols": [ + ["test_id", "INTEGER"], + ["title", "TEXT"], + ["json_id", "INTEGER REFERENCES json (json_id)"] + ], + "indexes": ["CREATE UNIQUE INDEX test_importfilter_id ON test_importfilter(test_id)"], + "schema_changed": 1426195822 + } + } + } + + if os.path.isfile(db_path): + os.unlink(db_path) + db = Db.Db(schema, db_path) + db.checkTables() + + def stop(): + db.close("Test db cleanup") + os.unlink(db_path) + + request.addfinalizer(stop) + return db + + +@pytest.fixture(params=["sslcrypto", "sslcrypto_fallback", "libsecp256k1"]) +def crypt_bitcoin_lib(request, monkeypatch): + monkeypatch.setattr(CryptBitcoin, "lib_verify_best", request.param) + CryptBitcoin.loadLib(request.param) + return CryptBitcoin + +@pytest.fixture(scope='function', autouse=True) +def logCaseStart(request): + global time_start + time_start = time.time() + logging.debug("---- Start test case: %s ----" % request._pyfuncitem) + yield None # Wait until all test done + + +# Workaround for pytest bug when logging in atexit/post-fixture handlers (I/O operation on closed file) +def workaroundPytestLogError(): + import _pytest.capture + write_original = _pytest.capture.EncodedFile.write + + def write_patched(obj, *args, **kwargs): + try: + write_original(obj, *args, **kwargs) + except ValueError as err: + if str(err) == "I/O operation on closed file": + pass + else: + raise err + + def flush_patched(obj, *args, **kwargs): + try: + obj.buffer.flush(*args, **kwargs) + except ValueError as err: + if str(err).startswith("I/O operation on closed file"): + pass + else: + raise err + + _pytest.capture.EncodedFile.write = write_patched + _pytest.capture.EncodedFile.flush = flush_patched + + +workaroundPytestLogError() + +@pytest.fixture(scope='session', autouse=True) +def disableLog(): + yield None # Wait until all test done + logging.getLogger('').setLevel(logging.getLevelName(logging.CRITICAL)) + diff --git a/src/Test/coverage.ini b/src/Test/coverage.ini new file mode 100644 index 00000000..ec34d0fc --- /dev/null +++ b/src/Test/coverage.ini @@ -0,0 +1,15 @@ +[run] +branch = True +concurrency = gevent +omit = + src/lib/* + src/Test/* + +[report] +exclude_lines = + pragma: no cover + if __name__ == .__main__.: + if config.debug: + if config.debug_socket: + if self.logging: + def __repr__ diff --git a/src/Test/pytest.ini b/src/Test/pytest.ini new file mode 100644 index 00000000..0ffb385f --- /dev/null +++ b/src/Test/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +python_files = Test*.py +addopts = -rsxX -v --durations=6 --capture=fd +markers = + slow: mark a tests as slow. + webtest: mark a test as a webtest. diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/content.json new file mode 100644 index 00000000..786db098 --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/content.json @@ -0,0 +1,133 @@ +{ + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "background-color": "white", + "description": "Blogging platform Demo", + "domain": "Blog.ZeroNetwork.bit", + "files": { + "css/all.css": { + "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", + "size": 112710 + }, + "data-default/data.json": { + "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", + "size": 196 + }, + "data-default/users/content-default.json": { + "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", + "size": 679 + }, + "data/data.json": { + "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", + "size": 31126 + }, + "data/img/autoupdate.png": { + "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", + "size": 24460 + }, + "data/img/direct_domains.png": { + "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", + "size": 16185 + }, + "data/img/domain.png": { + "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", + "size": 11881 + }, + "data/img/memory.png": { + "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", + "size": 12775 + }, + "data/img/multiuser.png": { + "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", + "size": 29480 + }, + "data/img/progressbar.png": { + "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", + "size": 13294 + }, + "data/img/slides.png": { + "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", + "size": 14439 + }, + "data/img/slots_memory.png": { + "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", + "size": 9488 + }, + "data/img/trayicon.png": { + "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", + "size": 19040 + }, + "dbschema.json": { + "sha512": "2e9466d8aa1f340c91203b4ddbe9b6669879616a1b8e9571058a74195937598d", + "size": 1527 + }, + "img/loading.gif": { + "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", + "size": 723 + }, + "index.html": { + "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", + "size": 4667 + }, + "js/all.js": { + "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", + "size": 201178 + } + }, + "files_optional": { + "data/img/zeroblog-comments.png": { + "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", + "size": 24001 + }, + "data/img/zeroid.png": { + "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", + "size": 18875 + }, + "data/img/zeroname.png": { + "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", + "size": 36031 + }, + "data/img/zerotalk-mark.png": { + "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", + "size": 44862 + }, + "data/img/zerotalk-upvote.png": { + "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", + "size": 41092 + }, + "data/img/zerotalk.png": { + "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", + "size": 26606 + }, + "data/optional.txt": { + "sha512": "c6f81db0e9f8206c971c9e5826e3ba823ffbb1a3a900f8047652a8bf78ea98fd", + "size": 6 + } + }, + "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*|data/test_include/.*)", + "includes": { + "data/test_include/content.json": { + "added": 1424976057, + "files_allowed": "data.json", + "includes_allowed": false, + "max_size": 20000, + "signers": ["15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo"], + "signers_required": 1, + "user_id": 47, + "user_name": "test" + }, + "data/users/content.json": { + "signers": ["1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f"], + "signers_required": 1 + } + }, + "inner_path": "content.json", + "modified": 1503257990, + "optional": "(data/img/zero.*|data/optional.*)", + "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G4Uq365UBliQG66ygip1jNGYqW6Eh9Mm7nLguDFqAgk/Hksq/ruqMf9rXv78mgUfPBvL2+XgDKYvFDtlykPFZxk=" + }, + "signs_required": 1, + "title": "ZeroBlog", + "zeronet_version": "0.5.7" +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/css/all.css b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/css/all.css new file mode 100644 index 00000000..c2ad65fc --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/css/all.css @@ -0,0 +1,385 @@ + + +/* ---- data/1Hb9rY98TNnA6TYeozJv4w36bqEiBn6x8Y/css/Comments.css ---- */ + + +.comments { margin-bottom: 60px } +.comment { background-color: white; padding: 25px 0px; margin: 1px; border-top: 1px solid #EEE } +.comment .user_name { font-size: 14px; font-weight: bold } +.comment .added { color: #AAA } +.comment .reply { color: #CCC; opacity: 0; -webkit-transition: opacity 0.3s; -moz-transition: opacity 0.3s; -o-transition: opacity 0.3s; -ms-transition: opacity 0.3s; transition: opacity 0.3s ; border: none } +.comment:hover .reply { opacity: 1 } +.comment .reply .icon { opacity: 0.3 } +.comment .reply:hover { border-bottom: none; color: #666 } +.comment .reply:hover .icon { opacity: 1 } +.comment .info { font-size: 12px; color: #AAA; margin-bottom: 7px } +.comment .info .score { margin-left: 5px } +.comment .comment-body { line-height: 1.5em; margin-top: 0.5em; margin-bottom: 0.5em } +.comment .comment-body p { margin-bottom: 0px; margin-top: 0.5em; } +.comment .comment-body p:first-child { margin: 0px; margin-top: 0px; } +.comment .comment-body.editor { margin-top: 0.5em !important; margin-bottom: 0.5em !important } +.comment .comment-body h1, .comment .body h2, .comment .body h3 { font-size: 110% } +.comment .comment-body blockquote { padding: 1px 15px; border-left: 2px solid #E7E7E7; margin: 0px; margin-top: 30px } +.comment .comment-body blockquote:first-child { margin-top: 0px } +.comment .comment-body blockquote p { margin: 0px; color: #999; font-size: 90% } +.comment .comment-body blockquote a { color: #333; font-weight: normal; border-bottom: 0px } +.comment .comment-body blockquote a:hover { border-bottom: 1px solid #999 } +.comment .editable-edit { margin-top: -5px } + +.comment-new { margin-bottom: 5px; border-top: 0px } +.comment-new .button-submit { + margin: 0px; font-weight: normal; padding: 5px 15px; display: inline-block; + background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; font-size: 15px; line-height: 30px +} +.comment-new h2 { margin-bottom: 25px } + +/* Input */ +.comment-new textarea { + line-height: 1.5em; width: 100%; padding: 10px; font-family: 'Roboto', sans-serif; font-size: 16px; + -webkit-transition: border 0.3s; -moz-transition: border 0.3s; -o-transition: border 0.3s; -ms-transition: border 0.3s; transition: border 0.3s ; border: 2px solid #eee; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow-y: auto +} +input.text:focus, textarea:focus { border-color: #5FC0EA; outline: none; background-color: white } + +.comment-nocert textarea { opacity: 0.5; pointer-events: none } +.comment-nocert .info { opacity: 0.1; pointer-events: none } +.comment-nocert .button-submit-comment { opacity: 0.1; pointer-events: none } +.comment-nocert .button.button-certselect { display: inherit } +.button.button-certselect { + position: absolute; left: 50%; white-space: nowrap; -webkit-transform: translateX(-50%); -moz-transform: translateX(-50%); -o-transform: translateX(-50%); -ms-transform: translateX(-50%); transform: translateX(-50%) ; z-index: 99; + margin-top: 13px; background-color: #007AFF; color: white; border-bottom-color: #3543F9; display: none +} +.button.button-certselect:hover { background-color: #3396FF; color: white; border-bottom-color: #5D68FF; } +.button.button-certselect:active { position: absolute; -webkit-transform: translateX(-50%) translateY(1px); -moz-transform: translateX(-50%) translateY(1px); -o-transform: translateX(-50%) translateY(1px); -ms-transform: translateX(-50%) translateY(1px); transform: translateX(-50%) translateY(1px) ; top: auto; } + +.user-size { font-size: 11px; margin-top: 6px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; text-transform: uppercase; display: inline-block; color: #AAA } +.user-size-used { position: absolute; color: #B10DC9; overflow: hidden; width: 40px; white-space: nowrap } + + + +/* ---- data/1Hb9rY98TNnA6TYeozJv4w36bqEiBn6x8Y/css/ZeroBlog.css ---- */ + + +/* Design based on medium */ + +body { background-color: white; color: #333332; margin: 10px; padding: 0px; font-family: 'Roboto', sans-serif; height: 15000px; overflow: hidden } +body.loaded { height: auto; overflow: auto } +h1, h2, h3, h4 { font-family: 'Roboto', sans-serif; font-weight: normal; margin: 0px; padding: 0px } +h1 { font-size: 32px; line-height: 1.2em; font-weight: bold; letter-spacing: -0.5px; margin-bottom: 5px } +h2 { margin-top: 3em } +h3 { font-size: 24px; margin-top: 2em } +h1 + h2, h2 + h3 { margin-top: inherit } + +p { margin-top: 0.9em; margin-bottom: 0.9em } +hr { margin: 20px 0px; border: none; border-bottom: 1px solid #eee; margin-left: auto; margin-right: auto; width: 120px; } +small { font-size: 80%; color: #999; } + +a { border-bottom: 1px solid #3498db; text-decoration: none; color: black; font-weight: bold } +a.nolink { border-bottom: none } +a:hover { color: #3498db } + +.button { + padding: 5px 10px; margin-left: 10px; background-color: #DDE0E0; border-bottom: 2px solid #999998; background-position: left center; + -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; text-decoration: none; -webkit-transition: all 0.5s ease-out; -moz-transition: all 0.5s ease-out; -o-transition: all 0.5s ease-out; -ms-transition: all 0.5s ease-out; transition: all 0.5s ease-out ; color: #333 +} +.button:hover { background-color: #FFF400; border-color: white; border-bottom: 2px solid #4D4D4C; -webkit-transition: none; -moz-transition: none; -o-transition: none; -ms-transition: none; transition: none ; color: inherit } +.button:active { position: relative; top: 1px } + +/*.button-delete { background-color: #e74c3c; border-bottom-color: #A83F34; color: white }*/ +.button-outline { background-color: white; color: #DDD; border: 1px solid #eee } + +.button-delete:hover { background-color: #FF5442; border: 1px solid #FF5442; color: white } +.button-ok:hover { background-color: #27AE60; border: 1px solid #27AE60; color: white } + +.button.loading { + color: rgba(0,0,0,0); background: #999 url(../img/loading.gif) no-repeat center center; + -webkit-transition: all 0.5s ease-out; -moz-transition: all 0.5s ease-out; -o-transition: all 0.5s ease-out; -ms-transition: all 0.5s ease-out; transition: all 0.5s ease-out ; pointer-events: none; border-bottom: 2px solid #666 +} + +.cancel { margin-left: 10px; font-size: 80%; color: #999; } + + +.template { display: none } + +/* Editable */ +.editable { outline: none } +.editable-edit:hover { opacity: 1; border: none; color: #333 } +.editable-edit { + opacity: 0; float: left; margin-top: 0px; margin-left: -40px; padding: 8px 20px; -webkit-transition: all 0.3s; -moz-transition: all 0.3s; -o-transition: all 0.3s; -ms-transition: all 0.3s; transition: all 0.3s ; width: 0px; display: inline-block; padding-right: 0px; + color: rgba(100,100,100,0.5); text-decoration: none; font-size: 18px; font-weight: normal; border: none; +} +/*.editing { white-space: pre-wrap; z-index: 1; position: relative; outline: 10000px solid rgba(255,255,255,0.9) !important; } +.editing p { margin: 0px; padding: 0px }*/ /* IE FIX */ +.editor { width: 100%; display: block; overflow :hidden; resize: none; border: none; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; z-index: 900; position: relative } +.editor:focus { border: 0px; outline-offset: 0px } + + +/* -- Editbar -- */ + +.bottombar { + display: none; position: fixed; padding: 10px 20px; opacity: 0; background-color: rgba(255,255,255,0.9); + right: 30px; bottom: 0px; z-index: 999; -webkit-transition: all 0.3s; -moz-transition: all 0.3s; -o-transition: all 0.3s; -ms-transition: all 0.3s; transition: all 0.3s ; transform: translateY(50px) +} +.bottombar.visible { -webkit-transform: translateY(0px); -moz-transform: translateY(0px); -o-transform: translateY(0px); -ms-transform: translateY(0px); transform: translateY(0px) ; opacity: 1 } +.publishbar { z-index: 990} +.publishbar.visible { display: inline-block; } +.editbar { -webkit-perspective: 900px ; -moz-perspective: 900px ; -o-perspective: 900px ; -ms-perspective: 900px ; perspective: 900px } +.markdown-help { + position: absolute; bottom: 30px; -webkit-transform: translateX(0px) rotateY(-40deg); -moz-transform: translateX(0px) rotateY(-40deg); -o-transform: translateX(0px) rotateY(-40deg); -ms-transform: translateX(0px) rotateY(-40deg); transform: translateX(0px) rotateY(-40deg) ; transform-origin: right; right: 0px; + list-style-type: none; background-color: rgba(255,255,255,0.9); padding: 10px; opacity: 0; -webkit-transition: all 0.6s; -moz-transition: all 0.6s; -o-transition: all 0.6s; -ms-transition: all 0.6s; transition: all 0.6s ; display: none +} +.markdown-help.visible { -webkit-transform: none; -moz-transform: none; -o-transform: none; -ms-transform: none; transform: none ; opacity: 1 } +.markdown-help li { margin: 10px 0px } +.markdown-help code { font-size: 100% } +.icon-help { border: 1px solid #EEE; padding: 2px; display: inline-block; width: 17px; text-align: center; font-size: 13px; margin-right: 6px; vertical-align: 1px } +.icon-help.active { background-color: #EEE } + +/* -- Left -- */ + +.left { float: left; position: absolute; width: 170px; padding-left: 60px; padding-right: 20px; margin-top: 60px; text-align: right } +.right { float: left; padding-left: 60px; margin-left: 240px; max-width: 650px; padding-right: 60px; padding-top: 60px } + +.left .avatar { + background-color: #F0F0F0; width: 60px; height: 60px; -webkit-border-radius: 100%; -moz-border-radius: 100%; -o-border-radius: 100%; -ms-border-radius: 100%; border-radius: 100% ; margin-bottom: 10px; + background-position: center center; background-size: 70%; display: inline-block; +} +.left h1 a.nolink { font-family: Tinos; display: inline-block; padding: 1px } +.left h1 a.editable-edit { float: none } +.left h2 { font-size: 15px; font-family: Tinos; line-height: 1.6em; color: #AAA; margin-top: 14px; letter-spacing: 0.2px } +.left ul, .left li { padding: 0px; margin: 0px; list-style-type: none; line-height: 2em } +.left hr { margin-left: 100px; margin-right: 0px; width: auto } +.left .links { width: 230px; margin-left: -60px } +.left .links.editor { text-align: left !important } + +/* -- Post -- */ + +.posts .new { display: none; position: absolute; top: -50px; margin-left: 0px; left: 50%; -webkit-transform: translateX(-50%) ; -moz-transform: translateX(-50%) ; -o-transform: translateX(-50%) ; -ms-transform: translateX(-50%) ; transform: translateX(-50%) } + +.posts, .post-full { display: none; position: relative; } +.page-main .posts { display: block } +.page-post.loaded .post-full { display: block; border-bottom: none } + + +.post { margin-bottom: 50px; padding-bottom: 50px; border-bottom: 1px solid #eee; min-width: 500px } +.post .title a { text-decoration: none; color: inherit; display: inline-block; border-bottom: none; font-weight: inherit } +.posts .title a:visited { color: #666969 } +.post .details { color: #BBB; margin-top: 5px; margin-bottom: 20px } +.post .details .comments-num { border: none; color: #BBB; font-weight: normal; } +.post .details .comments-num .num { border-bottom: 1px solid #eee; color: #000; } +.post .details .comments-num:hover .num { border-bottom: 1px solid #D6A1DE; } +.post .body { font-size: 21.5px; line-height: 1.6; font-family: Tinos; margin-top: 20px } + +.post .body h1 { text-align: center; margin-top: 50px } +.post .body h1:before { content: " "; border-top: 1px solid #EEE; width: 120px; display: block; margin-left: auto; margin-right: auto; margin-bottom: 50px; } + +.post .body p + ul { margin-top: -0.5em } +.post .body li { margin-top: 0.5em; margin-bottom: 0.5em } +.post .body hr:first-of-type { display: none } + +.post .body a img { margin-bottom: -8px } +.post .body img { max-width: 100% } + +code { + background-color: #f5f5f5; border: 1px solid #ccc; padding: 0px 5px; overflow: auto; -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; display: inline-block; + color: #444; font-weight: normal; font-size: 60%; vertical-align: text-bottom; border-bottom-width: 2px; +} +.post .body pre { table-layout: fixed; width: auto; display: table; white-space: normal; } +.post .body pre code { padding: 10px 20px; white-space: pre; max-width: 850px } + +blockquote { border-left: 3px solid #333; margin-left: 0px; padding-left: 1em } +/*.post .more { + display: inline-block; border: 1px solid #eee; padding: 10px 25px; -webkit-border-radius: 26px; -moz-border-radius: 26px; -o-border-radius: 26px; -ms-border-radius: 26px; border-radius: 26px ; font-size: 11px; color: #AAA; font-weight: normal; + left: 50%; position: relative; -webkit-transform: translateX(-50%); -moz-transform: translateX(-50%); -o-transform: translateX(-50%); -ms-transform: translateX(-50%); transform: translateX(-50%) ; +}*/ + +.post .more { border: 2px solid #333; padding: 10px 20px; font-size: 15px; margin-top: 30px; display: none; -webkit-transition: all 0.3s ; -moz-transition: all 0.3s ; -o-transition: all 0.3s ; -ms-transition: all 0.3s ; transition: all 0.3s } +.post .more .readmore { } +.post .more:hover { color: white; -webkit-box-shadow: inset 150px 0px 0px 0px #333; -moz-box-shadow: inset 150px 0px 0px 0px #333; -o-box-shadow: inset 150px 0px 0px 0px #333; -ms-box-shadow: inset 150px 0px 0px 0px #333; box-shadow: inset 150px 0px 0px 0px #333 ; } +.post .more:active { color: white; -webkit-box-shadow: inset 150px 0px 0px 0px #AF3BFF; -moz-box-shadow: inset 150px 0px 0px 0px #AF3BFF; -o-box-shadow: inset 150px 0px 0px 0px #AF3BFF; -ms-box-shadow: inset 150px 0px 0px 0px #AF3BFF; box-shadow: inset 150px 0px 0px 0px #AF3BFF ; -webkit-transition: none; -moz-transition: none; -o-transition: none; -ms-transition: none; transition: none ; border-color: #AF3BFF } + + + +/* ---- data/1Hb9rY98TNnA6TYeozJv4w36bqEiBn6x8Y/css/fonts.css ---- */ + + +/* Base64 encoder: http://www.motobit.com/util/base64-decoder-encoder.asp */ +/* Generated by Font Squirrel (http://www.fontsquirrel.com) on January 21, 2015 */ + +@font-face { + font-family: 'Tinos'; + src: url(data:application/x-font-woff;charset=utf-8;base64,d09GRgABAAAAAIfEABMAAAABK6AAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABqAAAABwAAAAcafTEEEdERUYAAAHEAAAAJAAAACYAJwFhR1BPUwAAAegAAAJ1AAAHzLT7y6ZHU1VCAAAEYAAAAIIAAADSeGF8IE9TLzIAAATkAAAAYAAAAGD/Bgk2Y21hcAAABUQAAAJGAAADdg18Ei5jdnQgAAAHjAAAADIAAAAyDwsIvWZwZ20AAAfAAAABsQAAAmVTtC+nZ2FzcAAACXQAAAAIAAAACAAAABBnbHlmAAAJfAAAceQAAQSsvLBIQGhlYWQAAHtgAAAAMwAAADYI0lERaGhlYQAAe5QAAAAhAAAAJA+wBn9obXR4AAB7uAAAAioAAATsC2JNKGxvY2EAAH3kAAACcAAAAnjTQBTGbWF4cAAAgFQAAAAgAAAAIAJYAbZuYW1lAACAdAAAAxwAAAd009XuX3Bvc3QAAIOQAAADlQAABiya+85BcHJlcAAAhygAAACUAAAAz1AoMgx3ZWJmAACHvAAAAAYAAAAGNWtUwAAAAAEAAAAA0MoNVwAAAADIRNDOAAAAANDl5ep42mNgZGBg4AFiMQY5BiYGRgZGRisgyQIUYQJiRggGAAysAIp42uVUv2tTURg9372vbSqaHyWUUkymh6i0Kg1SE4o4XEpaMsVGk/AGqwSDqcUGEbSlgyCIy4MiIuLgX5BBMjg4iDg5iJtCFwen4uDg0MnreTcZ/NmtRZFw3ne/75xz73fvzXsQAPuwgDV4l262lzB+ud1o4VCzcbGNqaXF68s4DY8aWItI+/ux/FBXrUZ7GfGri+0W0swFMT5jbqSgqR3AIIYwTPYgjuIUZtnBFdzDfaqi2e666OEBnuINtvrZtqTlhJR6mdTlhmxIp589l/fyRSVdFlNZVVCBWldP1Ev1UQ+66gG9Xx/WRl/Qa/qh7uq3+pMX83zvjBc4XnlV7xb749jbYJeCESIe7cvtKkJUTRNJYuy7eo9T5MYci1+43fDtBrfXe9jJt9ecxijGkXU3/zP7ryn+pnP9v7nonhTvafSPN7WzQpRyX9cRHIPBPK5hHY/RwSuJS1Fuywt5LR/cb0s+88vq2xDTdhN5okDM2G70BvI5DO24DPJft8kF5Axqtok6Y2CNTJLTSJBJET6ZIXq69BjOZ+hp0pOjNqTWsK+ITdCVInxWBqi9Q21AbUhtSG2HvWt2kqA7SU2KMWMfIUvWp3KKnLErmCWKRMk+Q5mxwniOscpYZwyIeG8m9u9mYsxwtizhc2zYRZEosYcy8wpjlQhY6zvpSnLdFGOGvWcJn6xhH0WixG7LjBXGKhHQnejt0q2Z6a9p6DR9Z47OHJ0hnTmcZX2B9SpRY64wZzdlEis8mwRnTNp3XD3EvKuWyCfsKiurEJngPqIzbXLfIc5HCo6jt1zzP+bjCI4jh5OYRh4FzGAOFdRQRyATMvkNmAz7ZgAAAHjaY2BkYGDgYghiyGFgSa4symGQSi9KzWZQyUhNKmIwyEksyWOwYWABqmH4/x9IwFiMKGxGFHGm5OTcAgY+MCkC5DOCRUGYmYGDQYBBgoENLAaiQeI6UHknoDxQN1CFCFiWAa4PohdECwGxFMQWIMnE4MPgC1XBxtALNtUHAMqqFmYAAAADBB0BkAAFAAQFmgUzAAABJQWaBTMAAAOgAGQBpAEFAgIGAwUEBQIDBOAACv9QAHj/AAAAIQAAAABNT05PAEAADSX8Bmb+ZgAAB9oClWAAAb/f9wAAA6wFPQAAACAABHjarZPpU81RGMc/z68FUaK0y6+oaN/rhsoeiksia7asWbNky9jXsQuhSZSkYgYzzRgaXvgTTJYX947/gGG8yD3O3HsnZpjxxpk55zzPmXM+58z3+R7AA1ePQPSIVOlMnLmnWPVspRAv/PGllrs084CHdPKULp7zlm8SIHGSIGmSI0VSImVSKdVSa4Qbb4z3xkfTxww0w8xIM9qMNVPMPLPCbI+KjulVSpN9MWnUxFZN7HATu+nhuwRJvCRLplikWKxSLhukxghxEzH9zRAzwk20/CKqr+qTeq1eqW71Ur1QXeqZeqIeq0eqU7WrNtWqWlSzalKNqkHVqzrHD0epo9BRYI+wB9sD7QF2f7uv3cvWa+uxnbUFf8h3qfFfm7fh41SYP9iC4Y6MfzBcJz3w1DXxph/9GYAPAxmk1fRjsK7TEIYSQCDDCCKYEEIJI1xXcziRjNCKRxHNSEYRQyxxjGYM8SSQSBLJpJBKGulkkEkW2eSQi4U8xjKO8eRToL0wgYlMYjJTmMo0ipjODGZSTAmzmK3dMoe5lDKPMuazgHIWsojFLGEpy6hgOSv0+3ewk93s4RDHOcMFznORy1ziCnVc5xo3qOcWN7lNg3ZIE3eczrunnXJf+6/NqcFqKrUc6To6x0bWiYUq1upsFyf61FrzFwWv0sIWVv22sp7NksFKtlLNMT7zRbsvQVIkVRIlybmjQ/z0XbmSJdl9hUiWND1tZy/b2EcNB/T/OMh+jnBUrx/mFKc5yTsJlXD3iTA2uaKffxyhPwAAAAADrAU9AFAALQA1AFYAWgBoAKYAaQCmALQAwQDRAGYAXwCEAI8AYQCaAJYAXABEBREAAHjaXVG7TltBEN0NDwOBxNggOdoUs5mQxnuhBQnE1Y1iZDuF5QhpN3KRi3EBH0CBRA3arxmgoaRImwYhF0h8Qj4hEjNriKI0Ozuzc86ZM0vKkap36WvPU+ckkMLdBs02/U5ItbMA96Tr642MtIMHWmxm9Mp1+/4LBpvRlDtqAOU9bykPGU07gVq0p/7R/AqG+/wf8zsYtDTT9NQ6CekhBOabcUuD7xnNussP+oLV4WIwMKSYpuIuP6ZS/rc052rLsLWR0byDMxH5yTRAU2ttBJr+1CHV83EUS5DLprE2mJiy/iQTwYXJdFVTtcz42sFdsrPoYIMqzYEH2MNWeQweDg8mFNK3JMosDRH2YqvECBGTHAo55dzJ/qRA+UgSxrxJSjvjhrUGxpHXwKA2T7P/PJtNbW8dwvhZHMF3vxlLOvjIhtoYEWI7YimACURCRlX5hhrPvSwG5FL7z0CUgOXxj3+dCLTu2EQ8l7V1DjFWCHp+29zyy4q7VrnOi0J3b6pqqNIpzftezr7HA54eC8NBY8Gbz/v+SoH6PCyuNGgOBEN6N3r/orXqiKu8Fz6yJ9O/sVoAAAAAAQAB//8AD3ja5L0JfFTluTh83nNmTWY7s2bWzJLJwpAMmckQIhACsomIioiIFkEpm+IKoqKiIiKiIq5IqSuipak9ZzKgorWo127WWv8W1Kq11npt7rVqe71WgRy+53nfcyaTZJKgvf3u//t9+iM5mSzzPvv+vBzPTeY4frH+dE7gjFyTTLj0uLxR5/trRjbo3x2XF3h45GQBX9bjy3mjoerIuDzB17NiTEzGxNhkPqrUkG3KMv3ph344WfcqB3+Se+foh+RV/XiukrNz53F5M8elCoKJE3WpvIXnUkRypCXuYMFgw5fUT102A2dKFewV3DZdSrKnCzb6JIskJds50SlZ2iSbKJuFtjbJ4JSsbZxsEeBlW9uo5taW0dmM1+M2JOK1rqyQeGdqNjt1SkvLFMvv2xecnZ06Nds8fbreeWQ6nG2dcDb/MpwNYe7g8vBKStJl8XhmeF9DhkimtCQcLPAV3Gh4gXfIRgJnpF/JZjiMkRedMtG1tXGjmvHNCPxb93L7DDIZPujHK9XkA6Wao3hIcpzwDrxXkKsmZ3D5AOAh7/H6s9ls3gjvmzdVWuC5wJGA0Zrq4sVQuMaXlTlTd5fbVxWs8WUKeh39luCIVOO39PAtg7nCCt8iUjQtBQ7K/opuyU8PKZsquvNGU0Wqq8OoM6ckLiOZHLIXvuGBb3i8+A2PHb7hzEgeh1wJ37BUdMsxkpJGB/a1f/hFhPOkKva1/8cXE/FBCji6+IDRBe9OPxrwI7xVl9lvggevo6vCWwkPHkeX1WOBH3DQjyL96MaP+DM++jPwW1X0t+BvBrW/E9L+Thh/piui/WQ1vi50OHgBQXWIiItQOFLd1O8/qSOAJMhlXYlcDCiB/7KeBPyLCQkX/muNuWLJ/ccdIpaZy04kuZnLZu7c2/aZcmTWslnKr05aPusakmtTXiG7lpDsUnKfshz/LVVeXaLMJ7vwH7wOdBS4eUcXCy/oX+HquVFcK/cTLl8LlJQNQrfUnMnXGhCxtUlzKh9HAruQoxxZOWjolpoyeVcQv+0SzakuX1I0AfOPSUvWg3KDuVsi0YOixmgNDjlJUnmDozGTyRRqKrgTQFqcwWb4SqpxyC1AJV9Gjpq7C17Gi21AvwYrSIDQJrfUwOcqEAlDLTxwbZJP3EOsoWhja42vTQo6pYY2ySVKHpQVMeuLEJ/YpMu1jG7NZT1en5ioayJ1IrwMImT0JHJNxOXGn7ER0k5yLbV18ybFq3905pqFM1vDr+6eff/2ieHIle3nPHbLm0/NPv/j9XUzFiw+n8QuvHzR+izp+GnDCXp+dEP6uFnfbb9jr+36dYak0r0zNEKnJOLju2768ZvWW2/mR+rEsaeNryPPWlccflm88bIFV2U4PTf16EeGxfrZnJnzcH6uBrB9O0eRKnmzcj2guykth+GTEfAmW+HBlZZ5+ITS0IwqRa4ArFY4ZAfgRg+PeoccgMckPCYdcgoeAYFyBj47KkRnl5H3+gFBcioJX/jC8Sr4gpON9fBVIJpM4bdcYfhCX+Hg4AtgNkcM9IwjFq9tdXuzmdGAm0Tc4CJZM2EqqP+3ppJpDz6m7H34oe+dMnf+nNPOOuPk5ULhoiMn8rFHdj24U3nq4Yd3aN8QPiQXPf2Ucuvze3du3LJj220bDi/XLzm0nVyY/wm8vPn5p9SXgR9nHf3E4AY81XCN3BjueoYjud7QnQ/jQ0jfLbnTEugLDzyNykij06A3u4nURnGUtIISSYKylZssFC8tqAks3fJx8LkJUCHp2qQWsWCO16ccyD8WpzQC8OIOic49OgcnVlPMjPaAHrQY2hAtLe2ktTanamEbMZLRrXW5mMfIG6uJJ1ZnIwkVLa3EBrzla+dzLYCdWdOvvum+/XuffPr4q7aQXYsMt5Nfbp655s+PKU/8cNmHG7/6+t9+8ODb9yvW5Qse2r7o9NTe5WeTxYtuPS97554t99y+adalZ4xRrvvZY8FcLqj8Q/qtP/e978x+4cWtO+4nv2zczn/30VU1UxbP2L+QI9xc4WwSpDo/zjS+qu6JpCvR9bKepFSdPldV5fC785WZ/DL9bZzI1XGSLQ2/QSQnxaPZ2g2mDaXbbAOtqsvILvYHWpyjWz02wosOp9fnaSL8/GV/+MVVdxw/9d5Lfvb+cvLpp+Tkr1bMmfqO8q5ySDmivPK7iXOWf01ORZtBuEnwfqdo7yemJTN7P91B2Qrvp7Pi++lAm0im4vu1E6fo4GvrchHi9Lh5g3HSJfdOPf6Oq37xh2XwxspDEw+QHBGIgdS8M3XO+f9Q5E8/VX781fns/Rbzs4QHADci/J/XU+vsTGt/uZG06oWskPRZiTHpSrj0i8l05ekOMja1I03GdygFMrNDeSm9I6W8LJx73uItL5JpyjPP33buotueU54l01+Cv7+Ee0VXrdsAvsBsTuLSkjErExNIZibPEYSFqwDFSSgaiWCGd7ekpYqDEp+RzWDZdJm8uYKi2Ag/VmHGxwrOnJKt7IC5mAieiCcmJsQl5ImlZLcydym/egl+XqLMJbsZjLOUl8k67g3OwTVx4HvIHAoD4JY/WLBWcFFQtDzFK+8ALc3x8Fayk/59H4pxXZGzDcZZUiQ5eXKu4+QH7/x0zMXuqCvRMbrt7KmbnhmB7zOT3Msv5NPAZ/UIa4GYuCA4OOwTMpvMgS8h2DhTL7eBmMzk3eTe++9nZ90IvtOV3AHAV66v51TyTJHU34FSUVLqC23U/CDNB4K/zx39hG/TT4UzRkEW0GARSnQd/Yt86dk8JEvIm+cqKbd+/qFdIDegd4Q86J0KzstN4vImVDY2AZWNLCBKfVQuKkGrVDqo42YArVIFn8VK0BUmcN1Ai9jg0cBRteFwZjPAurE4rz3yiTg/6613f/euMv13bz296robr77qphsu5XeSk8gi5VHlxz3TyWIyR+lU/pMkyFQyiYSULxjeCmCoN+inw9mmMLhkAsrPnMkbKHiVSG3ZZO7Om3gktQnZjqfOEs8x6bWgCwXOnWRA1y6bS4r6XDIr6j0F4lL+Rja2EGcuqJu6KPeHw6ZgDt9zPrznFv1MLsRdyd6zEKCkzovwnrIZ9LFZpMxrRcYOpyUbZTg091aHbID3c4NMu6kD4Q7BgQxufDQgg0cQfWDe82JVoA0RF0AnmLRJZlE2uMEJhq/sbXjOVtSmaGzsJCEWVbDBGPPM39f1xAmX35S7OJWYtHfdu++dUfjtmYv5/F0/+P6Lv9lww83hqp2ETz31+MU/f1meedZ2hsdzgcYSwDSCe4rL1yMe9UJ3Xl+PB9MTOGMYYXMDbO4wPXeVOVWwWurD1hSaZCKl0pL/YKGKgtlFwO1LFTgGM0oEe6pyyCa0y9T+SNVozkV4FtNytaU7L1bjHxYtgIWRyDxVorPA68Pxemqe9WCewR5JFlFKtklWpxRvk9yiVN0mhZ1SANkKHRxmgdMEsEN6sdNOslERxJh6Ool43bmTGuu2tH//3lu2btlyyQWrr8utGpmYdN4d08iT99+6t/OrD5/fTZqf80bveOLGTUbTbLPhuhs2rqVYC4jK3s2PuT2P37Pn1ynq6wOqhD/qJwOoNi3myfNoarhKM2+lMQZn4sJqjGEHxX5QsmSQIyUhkzdRzWYyAH7NlCnNyAPoxpjMAC4PcFeq9OdFCH5QcZCsCO4uaD6IwxL1/JbL9u7tVILkIz2p2yBsO3LpFuVtUreFf5TRdQ2l63guwj3B5UNFuoaKdHXgYV1MY3ktIQfQ0wtUttC4wWJDDq5OS/aDBXcFdzHwuNtOye8ElUnc9nJUdjMq+0EJRLUoLtImuwmFSDKJXXqbN8SIGqLfQ6L62ySvU3L3IWQTOBDGBFX0Re1Wt2ZS48i1Yx64p+PWy886YyH/RE/+grV/+fDLj56/B4hWFblHvnrduCr+/vuVOVXPv/Srt1MsJpsHeHgJ+Bv9S/CcfIgJ8CnzOmRrCwKsowCbgK3FhE8HaBCNwNZJqtsCwKX2DIQxFCK3pbvL5I4C7EbmQdSihxkAj9GiE33oOplEyQzgJcCLlDkqtSJIrayzUDXDfIVsxudBDxxUn1CEjmlE5kXOu/bDe+7cOikRu7jxzkdNj+6U973x8z999vQtm6+8YOnFm/h73yLH7Ux8vMkZVD5SPp/78+de/5AsJFOU15R3vr9n551oU4A3MeY1YcSr13R+wUiJ3cXpCYYlZmoATBQQUO2UH8EgyxVgC2RipIEGHBiIAKo3ZtwotPfsvp4/sacroePI+JMOO3Q7dzLflNoIL/haTdzFXN6DGA6AlYim5VqIiUaqzmiaItQHCDWg/CfgweeQw4BBJzw24GvolI7CMCcBnqdZ8ASidsotiGLZCTZEqhVlO2YERva6oqNbm0ivwWaSHnWVuuslz7OuW3nGylf273v1/LMuvebgn5Q5hctv2LDumo03rmpYvPLi5ResXLGUrNjw44b6bRft2rvn8Qvvrx/Rdd3z+8m1P7hp886H7thEPtp85dpNt9xwLeWtSQB7J/BWFcC+nukAWQTeEql0i2AF8gKyWUyAKC5kFlDEkLcSFBUgJ10WPyZBVBsBYb0bwr5IhtrRGhQlK814gL7bYxYFbyhG0SGiiqhsk0NeEQVHiomagUU0gFXlQFpyTJicLkPRF0drO+lPv/nogPLpV4/ePDERvaD1oU7ztu/Lrzx9xbXXr79y4yFh56/+oOxVHlJ+rHwv/pc77FXET8TT33jm/jtB+5BqhPkR8FfuEl4Af8XHje3rsdgJPHtUj6UKYQSNp3orklN7kv0D/Bax5PmRqdnM1CnZlilTWrJTpmayU89GlwZ8GSEMH5rBsWFyvQJ8vB1wDgdY4wVc3k2YNca30jy+cInHBzHjAKcPjHJBZKcDS4QGmBeB0dwBzfzKFVZE6wCv0FVy3hVytGbylJaOUx68a4x2ZHAVPVFnfOLotrOmgauoW188OfrH+8A/vpczcByodI+ZeJYIzxx5U2jgt2TIgaXKlcqVSxE+cG911cITNG8VQO+SutHgUupN3ZizwtyU6k8S+Id/ZJrwDHliyRKyfckSZgOWwXvF2Hu15swQFHiWCQ3wXs/sW0o2ko1LlVSG4fLoh0Ib8DHK8LtcPsoVccn41wVq0iXQvIYX7QLlXzUvIXlpaqlQQTMYXWKlz5YqhBlWtciccTNmnaz6F87AZJNNsjgk237Z7PhaMu7vMpkxR2RxdFktNlcqDx+jt0RvSRjAd2wDWwnKqI2DH7JYMQ9Eik9SR4DIPpQCD3gDoiy42trkCvQ9A1GkoQv1hr4C9IUggilFbI2mDoGdCAlB8xTspA89H9hEnnh4e5v1lMCWubNW3HTx7MRqjSV1CSWldD76s6hVOUJmhcVbb7rryuadR9pUDqW4XKmsNfxdP4dr507g9nPS2LTcAkoQ3FFntpA0cQmwthNNXDVgZ0paDgJJIxlUkTpk2BkUr6PM3Ej49iiH7AG8TqigPzzBITcQandiIEInqjm8O/++mabupjTZpMn79XJE/NomVe/n5OrJiKdI9eQpRTw1TBCdT+kqnfGRLWPGUj1S2QLoaR0D6EmijyFPmQg/YeQ88YZRE+xq/sPJxaKc6OZ1iXiupZUG9rmWdmEsyfgEG+haVK3OXAsXi+t40e3UwZetPgP8dA2fdOMXrrq4gV95mDxLbOQUcsmLyt6uiaYznjh++snjL3r8oRtqakddarInk6v2X6z8Qvm0W9n85vdI7S/u/npD7Dblg10fKD94lh97ypi7xl/WfMOPl5MVJER+TXjleeUPzyvyr7KZE884c+VZaz6U1jYaej5NXOpP+h8l8e1HSNWHymnK4ZeU3/7wpHOyF57zEzLvsXvaW3iP7kzlK0onN8fpn4b4wc45uYnMm6exqr47bzDbMpkMZf2CxcGh2rag2nalUT9zEDgA3ng98JrgAN5zMq8cFKTPE8u1CjHB54EP7m3kl8+Td3b3/LKwoefzjQU7eeOvEE98hcFELkjWK9cEc3wjvxkkfOzRQ7q/g712cWGultvM5V1oSTDT5c6wZ3osFMVaI/Uf6bNO312IhF0GOF8Ez1dHYx4IMYCn8jwNLXg1MQyumR9dMlu3FEdLa7Z1y/XwQtyNSR0vCkuYpxGGFBEx9V7rBEeGwiXW5WLRom3VxZKtqjlNkZz2MLazk/8uRJpb1+/5mfLfRzllzspbdvxw93Ovff7OI/dv/9FrAPcDgdx9Oy/fFXL96OYXfm6Yb9h6/23r5193x9qLQU/NOfqJ7iPQP0H00PxFK+qnVtSjWdFKobvL6BfQdwlRA2oDaGyl+XrJ4JC8mGP1sC89aS15im6GEbRJXnD7aXQl+gFacNiMoszZkJKgNSQ9ZXkulvG5qIMWxZQOwDiBZAVwKrg5ZAGZ9/rnhkmRqc8sVI5+8o9PLv3V2OREw7s9yj7lbn4LmU/OyyrvPJlKK/9HeUl5V/l1a9MvlJcnkPOYzRp79Ih+NdDZCrbzFC5v1ahsy7DnIpV9falMrSmQ1malALsAGCQgGlLZh2QrJRUH9p8RCT2BWAJps4hcRpYojys/UOIvXPHYl93K68rHBUYUZY9SUCRll26BicTJKDD4cWo7xnKc7hpay/iJGl97jN3FQFvL4PiBOHrqScM39RpTWt2cHsNDeM1t1aKGvJVypNUO5tfE6VUyAlQegMoCDOspCc6xYOEEJWfPFERGS5EGF6rjLRkpPxcCvdQ1iRQPQFCaGPewoKOIFjFRB3wqxlqz8BQTY1GkKaBG+HRS9OLfvUNWVHd0VCv3EhPhT2mb5FJx89O3bT1fPKQsfqTnPcdXygMaXhoALz7uIfA6KF703eDdUBdbNgHI9IEDNDjMbgJocMBrDuoROmwY7zlovFcJaNC5qScOxPUelEVAgymTF2n8JbrgJ700meBFjAD0NkBHZaYkn6DD3BJDAMq2zko5mJMdHnjwtrFsWV+wqYISx2pAb64eP75aWf1+zwsP6k0M2tcptF+iitJReEE29RtBNqPc97l8RJNNTSALXl+EurWAA3+myxih4hkrFU9QmXLQ2p0PUvIHkWPctEbiBl+iyxB0m3qpmtYKb3GMtlBeK4UIyCtS1QSgeZHIwTbZGMH4qozcin3kllLeYxTBbKHwLiKnf/Hh8cEpL5x/lPvrF5/NeXpUJ/ls/apDd6jSu5B8Z5zyx13NOQilfqa8rbxWHyLXBEaPDigza1vIQk6TC/1mKhcPq1bDxayG0xNAq4FCIFmzKAd5o8WOxR1zFRUHM+poxvJ+Rms/TU35g2ZMEuR5f6kA+GkpT6rIIGOUcAUqC5Ux3PCVJYPcz8lV1CaJgBEdRAdgmNp6eR8wEFM/MwZARkC14O/k13TyVZ2dPX/p7NnUyZg+F+j5G2/Hz4fPRTbgcz2vYPoLYJ8BSuxcWuedVpJzq8gUJV8yZmiuklez5bzG6hQ+c2n2jWUve0/o88zopGfRToHvDu+ZOvoJj7bRza1U7aIBvFGDiybOeGCmSnzvCnhvK7y3Jy2R0lou5iPscBI7TbLYMSIz2bV8IK2V0gygC0tpLlZKqxBZjVkNoHLq+dzo+6QmRVxTR9zayeu2zX586XUVX2z2BZ4XPsXjHnnxpl+uVvkjD+f1Y1aohD9kZ5XGHRaVOyrRzSBSgOLLx1jCR1HmAymBj5gD532lWAsirXVIa7OnL4UJ/WRGH4QYe+l7wway83vKaH78VuXpnv/as79I45v5K/Cz0tETBz9k11ZlLMtxon5bBue3azlOmtfsr/etmt7XiO7oT3R69lKKS2YHEh3r/KPVOr/NSl0ODmIZliLSiTTpUMK3mVYX1dP8uNeI6cDi9onOIn8scxOfsk49s+EDOHMVd4t6Zpsry6TS6vAVpdJE8Q78SiQ/Pa63ojvvpcf1VuFxvaXH9bKUtgVAslB5s5gxZAQNbaGyCHaMFhtFLya4TehUIGHyRidQphcIM+mVO5IgjDLkXlIg75LrQPL+WOjpTige1Rof0QlHNMHTNR9ep7v+8GtFuui3UR9xo8pXNgohmh4KXQWFTjZbkKfcFD4R4BP5oj3hxVL4RNZEYIOfsRnxGzYL/IzRRvsJAFSMPmSjSBNXkk2kdGIAWh2lABKVVL2w8fkC+WXnoiMva0BVC3dSijl1ew9vp7qEp3blMrArWFuY3FtboDlU2V1SXbAVqwt6W2l1gS9WF/RqdUH1faJcaXVhDij2yWQmuDj7lRcU6fW3/v2j37/97x+/yT/EdyqPKT9WdiuPkO+QM5SfKH8AB+h4MpEElY9Ufw1kIUZloYq7nMtb8IxWpvHoMauMmB1kz2jwTQ4Lb6VOAOUwGxUIyZHJm22aIszbzBqu0S83M/uos3YjL3GyiSV85CoLLctSG44Ba4qoLOQqutzkEPnk6+77DvS88uALy37w8Pa956INX/2bg8okZr17rtr+5GO3U1wrBf1Omqeq4RZzeS8tGmu4joMD6kjLlSbEc8Gq576D6RuW/vTbaLNJtYp8THdWg+Pcxbu9Rkx36kXJAVQIYRhkbJPiYl9a6FGDYmk0EW8idehB96fMJDJL+X5m3tXjZvvWpaddePH6mlHKT5U8pdI7f/73t/gH+CeBSj9SPHe2X7hAN6kimF66S0fOIacBvf5IImQKUCyofFikl/40oJeHi3C3cXknSkllVnaBWFQ4MiqhIFSSPCVEs3mdSDQbEq06LbkPosGQvJm8nTosdtFcmvtGotlZl4HeyvLcATvwoBUUAKLDAuiwuVA/u8EWR5yad5KLsdQR1c+JAcTkjz/y+Z3Xk/O3AvQb//ZKz4EHX5C77rvn2R8CTXc/d9NLtT1/AS09U9GppH16wwP3X09t8qajnwhfAG0buSvU2Cmu+WcVhmLA1ESzcSEwu2aIGYsuZBKO7zXDT3mTpmLTUxrlnsNYSWSxUhxjpRA6YV2c6A0j4SuwIwuzKnot7YiuF2ZYPe6IgWVS6sTefL5h0+cHHnl+YmLJvTfdN/68dRvWnTf+0z9e8PvTJyaSe+bd+v3x51274drzxgtfPrQ3pXTvnLFy4cS540emx561fuGLB2pjZPSe9PQt105fMLGxaczcdRTuJMjmC9QvuFiNpGwgmZWqHy4a0Jp2cVZwsCUhSx0DjpYy1EKFHYCsANVeQSlbgarPXqH5CJi4KvGuvSiZHGqaCnsby4lTP7MolJhDSup2TA5GfnVF84Geew5ke74T1PFkrPLy68bKw+dTot3nfp1j/ox+B7VVV6jxg9HEYgb0GoEh8wL1GqkG8R1ELsvrqQ+gF+CQPj31D1CH6x2Yq2dKvkIzSfkKUQOIMmmFnuaI4fxGOL+jqo0xI+hrRwwoozk3ukS0JiemwP8i5+1/FdW38o8jyn8r/wD+A9/mHeWuvV1CA/Vz3v/b5397T2hBWHxHP9BjjcnGjWQ0YHCYVDiMFRQOexqzgHACa68woKdC3xpMoo+PFMAg3iORwl09H2V73mfvmRTewfc79F96K74Xf/QD4yZK7/OYHyjZs+ztKtW3q7DSt/OkqeM8OrBv3F8/WUrTY/YmGyYbTcavJfN+sDVmm5pGtNm19BgcD6Q2rzeIbW29R8wWT2mGg4LKeJlMIMsbydlk1OvkudXKjJyyU/leTpnMjnyy8MiRBYLMji3sOLIYjw5nbwS99AiNGcapttvNvBOXl8YMBpX6enjJpFI/lGaOvR6dvYoq1eCaNTev19vDF32exrvJL5S19/ENu4mXtO4iT92tbCSvbut5c7fykHLVLnDjF/Ad1OE7l1D3SanomUfde1PPV9QkIz3hnHdRHM/j8g4a6+lpdqIcWUGinKxJxerUAvu8k4b7TofqXFudakzKieVor7lFPr61QK4nF+4GDlCmogApDRSjPffw51NvNa500KzZPv7PWLMG2b8MzulE2af+XiX4qJXUNa10oI+POkBA9ceZUP250pLxoKrf8gbq7RgqMCqhCWo1xUAtscOAcVQlq9w5sTuskpVhQddVUBoQlHasTZIYpjezwrn854f+8Z3qKcLWnkoSX/VZ89ipwQW6Te43yPi5h1d6KwmnbGb43aZ8yi/U38YZuTQHwa5s0HVLQlrmdTSjjwZXbwZpp3DosUCBXTEszQ/vpxf1nm1A5TdJg/Kp4cF0wyGxIU3/bp8eDwHlkBusx0PMksS55M3F+qmHdrGYDs7kLDkTx86kU89EDsoGOBOhPQsEzyRoZ/IlxWQuJsZmwHnehHON+bBB/1lD+utF9O+GyQZhF9DIgH1GrOUE/rqJQUOyKFFh0tlMOpcqj4AHtkF4+sh0LE7A7x79XPlK5zi6AODxcXgYztSN/2hHDQPDGPPEHLro4T8+sgRr3bpzeIf+Dk6PfTj6dIE3cTZ0YgwUfsHGVQD8RpQmXk+pydpIPYk1B/6tVX9+Uvk8Rv2H6WBLLxFeADlNcjex/l05AtqZOkweXXfeQTAeAvuqJnFAyxZ0iYDDCp8E7jJ8z1pKx5AZc1pSyIH5mjga2zR8QXtv9RnZbe6W64DZYiHMcDiw30Nyi7KvCq1twgMOlpurirEMfEs7qBy1TpHI0ToFig14U3EXTbCLRhuZfsJD226bNmPxafExj6zdvnW1orgulf+2a8vk8zLrbzqH6BasnSjo7lp0dnr1q4lrb+jZ4G9ccCmJSNVVZJZOv4DCztE6LvahhbnlqkXFrg8jghwA5xArUMgUkbTkOCg7AUA+k3dQ0XegOXI6ZB8aHEs39Rd9IPqy0UqddFT8jjYpgCIkGVERSJwoiQCeCAzkNrLqC099BldvsdZI0obF2/a/+sGBua/N2jVn9S1XX3fT2FUB/cJGRRp9x6zuT3qUr2ur9b7DFY63Dj5d8Fho7xfAMUt4GaKKhWot2g4UFBCKCiBcBS1jVRixjMUCDIsFSFXMIgOtaIxhtKADZPdQB8juoWl+qUIERYbVJGd/vwd8dKywoqsbndX99saHWlOnrVK+ePSHd17Q3DlfaSXv/WdPtXJoV1pZ9rs9sXE1DRTnM+CsHwPOMe+/ivmrsg9Oa6OBhk49NnjqkjktV6jIhyO74cgso4/cZAH59NOY1I9hGyLfj2l9wdmGDRay3YEwhLC3yMJh6AYcDPCYWDsCQFA7HvWxylWsdN4qAkwzfv+7Cy4yPEAmr1K+slxyn7KucN2Nl8w/81LlCHn1H4T4Yhu+qGo89Iy/cep88sHzz9bxH4vMDz8F4JoKNPBwIW6JmrWxaFQIYNbQ6KJZQ7VKCyCZLOhzYwO7T60YYj3WZ0IqWFxt1LlWIbG4GDsZMSbF3hFOpUVU9MTA/USXkwUdta4YZaRTfvNTsrzn+fHrlo6+e9Sox+a+88pecufZ5yy7iLz35d+VJbOdt9x/irNyTHXTJ+TUrvufoPprAgCyW/8I6IIruXwVQuDX0bBAJEwy1LIKyESeR5NuTWOHeN5KNaW1wszy2wHa2ugx0zDQE6A9+S5M7nh8YKcCNOcd4PCFQJU5xUw+toEx8uRaadEZ+0UQjjDBBiBmNCc88tg9J01qboyPmtBy6NArim6zMK+5btJv/+h6da3n4u075hz5MtbYGAMXVuDmKVOFt3VBro5r4To4ictnsMIbNnEBnNBAeNoFptwg7pNTQCDfmAw2BPmQQBNp75IhI+eARhyWV+rN3BjQaTmHPFbtimlyjzWBPgO9NgleyZmRbEaq18aKHWa7xR+OJFMZ2iLjdlK7mkkBi8awpyOMSR/J55QN8AvSGFE2QtghtTvzTreZOmM+1uOhdRz7Wn1GtSXICMSOsQa4RGkjdgy7jnsVybzpC9tmPfDAS/uSq6rfS2w+/sW9s6dOEKaPJt5td63+08P7f3nb1dc9vmfqNOXuEblJC5evPGvBsuXnvplbOsV1di4/4+3tT9grLkrdceKdj+yyrxcjN65avuP0y9fOPfHqi9o6ai4SQjdecc26DZdcymw8uPTCRyDTYe4+ZkEwTxPUYSaqCn09EzYquGkhuFSlhg7KPqZSfSEtA5gPUb8/RNkjFAAd61NzUuZiTgorCiUJKRR+0UfZB/RuEIt7oTbaloQ61ymF1bKA29hONFMCzNSSiGs5KjfJGKc/NeGqqy9Vzr9m14L165TFa1DjLl/W1DDu9o099/obG/38gs5wjwuf9Dy6GiArTohBsY//Ii0zrKP5TKvaZeYCTtMjp1HHwkvTU0YmFUZeSznleaOWoVJLl0aIcWzdUkWaqgU6imN1aak2yVhS3/F5SEkVP/XQBpJ6WLmjY2RjR0fjyI5GvxD1Nx7Z7m/Uvc9e6YCI4mVlJtkE50abdzVYCzxrkJ21glp6gMFOa/N5HY0TdQTFmiphbHpDw05YGg1IIlUBLBYNFrQpTji6M02J4kQ3Xgfho2zBQMwTpEDYAQg0g8asj3IrhYRydmlLwth0MJYZO+42kr3vzvbT/pOeHuB6Z2Xo2luF/QBW40u/chqu1+AiIOecjgMe9HMXsGwPBhc+4EGL3U3jDQROTwlU5MFAWqo6KLsYD7qqaLMHll6raKa9ipKkygssRlPQeh/ND1ShNeHAZZb8KmP1CUyIxlR1pJlcfy6ZuVr5gsxboqybqyhrFyvrkLGOjCEPBhsbfcqnPZ/6gKHIPRuVv6uchfIElhJ8u9uAt6b14SzqC+LBKzReKs9ImDWv1LLm/Tim4aGHyNSH8a2YHaPxOccZ/GC7mrnfaXXPJOaSdd0Ft6+mblSNL0OZQ3JkCyNNlAWCmRLWztDjpOE4aXqcdDMeJ12aeU07aO09AT+ToH22CRwmqk/gY30t/EzCQX0QIEYZSiANsvDdNAqDwQjSXU/9qgSIRhS+qhJlRzVy10haEgJpkXQiaGNradLWR8WffvSTog6w8cZelVDLdEJqzRriMp34RMeU1eGTXpv8+ZXK6bc+FJgyZYJHvE2ZtPn00+etv41J18zUcS1tqUnKf6o6Yl6nqcKqGz1R+/K0M8M9fkAygUiDEz4AHIdQR9J42EExnLc7g8ifiN+Cj+rIEszSdi45AEgJUMwGsIOaD5RiNsCK4zb4GRvFlg2jQZdNG8miTkWgiDgXIo4D1YKIslFEyYJxAJr6oyR9ySXEYj7pyeOYkjz19Pnr16koGNlWVJHzTjuH6kitjx7g7Z/rpv6Q+9t00pefPpr11rsHf4+d9IU119943bU3rV9N3vn0yD/+SxGVv334k397960X91H/LKbMFCQ4j4+LcbdyebvasZk343mqwVHT4YMAcuZjAaEH4v04DQYxdDFgJctNtBpuntBEKQGHBh2ikDpdlEDXgNCwXwqJstlOPWnsqiPYGBvAAECqxjlPWfBp7bGlWMeAoM4bphlTgJGhP7bmfGLiR9x2QuHlN1+5ZIlhp9JBQmeuOHLNutWzlLVVujf8ja01Iw/9x2fKIe/0BqUqzXMi2bX/2Rj1twHud4S3wd+OcNeW+NuUDtgUjIpRqkzLFh2OEtB8cK+7bQV3O5BhMwXW4kyBlVYWrZXAXJgMthY9b5Mo66m/6rQgCUH85DASk3O0FT3v1qzGWq3FplXmer/9+sUXzZ+6TLnk+orzHwbPe92GsZeFqev9xt/A9U4d2leVTlcR/aKpp5GPf7rfbeX/Khb15TagrZu7tERfulShstGW9KJQ5W00cWij8woeKl8Q2OWdVL7A/QL5cpYqUszpODXx4WRbsWw3QGzGEvBZIYRDLQsaxGke9+o4e+q4zKbMw0xUVmfOVi4MensmodplfQb8G3DueuwBov52FAgSpeovGsYeoHIhXAPNawczBauZm8xy2tg6AP62PAKpgewWqW3D0sQeE9iraBJdUA9EpnoEIFpF+wloYOfpH9h5tIQ29jioKe0cNdS5ljRp4uf84I6l30uPyF166uknzbpr/oSE/KOpPxjV2HziFS/ef+Z3z87w+67eUv3qyvQpk9tmBTJNx8+dcOsdYefH6yfvuqolHJt8nuovAtyNuutBM1yociRGSpRAJhZkmFiQQSijER0SheNY+Oo5KJvNbGrKU6y6emiFyYMDBahBzB51oMDiZA1eJlbgduUmQDiBAUUxdI3X5kT3Nd+d8OabY0cdd1rixjELzxHmNdYdODCnZ93ESY6JVdWXLuXvYucOgk57T+iEc1/BIryC3sTVAG8Vg1dCWFWSU4WJnVcA/vII2gkloeg+lfhOLAIXaGG+N4LVW9GdhUdMHmj9LW6MVbXoFUHwkyCJPHT1BbeT7BrlU9O0fRM+F9Y19izsrOY/Zh7gzLo24uaXsLyZjgMYSvLXRE106obIX4t989dk7z3ESQz3kLPnKxtnKNfA2zUGe9bwm8EMeI7cIFzFsfz1J/rn4b3c4Fn05q+Jmr/W9c9fs3y0bBKL2eggyQa196zEbDT56Y73P5zzpwPbyZJlysuz/vLRDOVZ9t4b+O/23Mdfz96fv6DnbvUMwF7Cx3CGvnloXW8emqh5aN0x5qGDfdy9SvjkXElyyucXkd3X7Pnt9WT2BcpXJHehMveat5XrGx1kG9kKhwoqfycOPJyyW2nygO9HTlJkTyPjKz/Eql/BGUPYlTmQHFiqRJXtowk4n5sNWqFdBzvOujGBPWR9ZQneGNJsPOuKB2ZvJ63ETwp3K1/eRRacq2yduHDVibNmpmPVrS3zTmhVtjMsXsTfRTF41mN3zHC+stA37vJtwsX0jK2AxzlwRhe3RtWxFtBLFpq3tWC9kBpSAS08lUXBhMd0pyXTQWx1Ap9Rm9uvZDEOWPyCkyWfsOouYojAW4DTnSLtTbGwWSCWuKnU0tA44pRjeWjsZmvlde+NWeauTvN7e44QV9u1gWza3yikXY6Nh5qPHAi6X1aeYzjep3wqhMGnjnNzOSQwZjZYHICRllHHRhSqD2Ku12vGzpq8l85oeYNsELWGMqgfAxlwNSUiyiKavAoIcWwM8RDLtIxGIwAKFCewPBECn730FdAyLbX73rxh/cb2s05aNu/MpSed1b7x2o3L+YZG3V9I7W3b0itXKHvP2eQSPBsXKfkVK9PbbyV1q1fTs29XphKMbTA3rGa4ZaGim/5TRyzFrLh9iTLVOO2rZ8rAG2cuTYCBLaZlpwqvjg5oRcw0y6OL0Hgvjr13Krx0IkAfATj94M6Y6Ege7Z0d1eyjwRvO1zcRBI62xoLf6PaBWvXQ+v0+gHXJ/HnLANabb7jhzQvO33jNzR+uXEFmLdroEVy3LCAzAM5ttym//4uuUbfqcuXtzd+jZ28jX+r8wiyI5abS+WAuC+8NR0zLgglzzDRy44p7IXCo0QqoCKJSRQIJIupOLzaFGVyUNsz1Z2myLJUKnEUCi+7ccPmYNSfOPS0xZZZrqdixYvaKzTMW1Z40y0q+fPb7x02Y3rx80w23zLxyQtvKG1me7x1lCXmVzoLYAbPUQhfM6vzH4Cs/rGzlh7XPyg/rwJUfdrpxoyQOBg3oegfHW3HM1dK+oF2Yrg5WHD4Vxx/omdaBbK7Tj+cCXBS7gXDei856FTw2HO6SIlnZY8JYHfxbR3R/WjJk5WqB8ng1NUvVNHXHWhiD4PkFWUpSiGQy2E18ApZSQFNjf1AFiK2ZiW0cOcSGI/JOzF5VFzPFMm/S5pS05QtYGacGl+1eAA/eJxrWvfGz2RvXtKYmzffdeUHi9sWzX/tV4cI1H/Gd7/VUdu42VCmHHqupOPJBbFwoZdy3z/rVX/N7qgRPmPLIlQDzC/qZXJLbw3o0MYuBc6m4dUS2GyCwovGwHesVHIkYrbQrjROYZ19Lx82xzVBHk0o6DPbjog5oVVXB3Qh/pypOQ08PTQKEWT8qcFqhkhXOsJYR14nOPB9MYkagCsdoXF6WfjZSxz8YAfpWU9Mdb5PthG0dMLKimiuX1XJ7rIsABCfmidGGiRhaFjqAc+W641//7ZjL10xMfGfxR2HiVQ45pp/B54/sm74l5uVXz96x+Ym99pCi7MwpX256YPbK8WfePmP76o6rr0mfO5rpAfCbBAV4o4a7Q8vp6qm0D6xvFkLhKgJoErM0/+7KFGJxfEHWW7JZHOiSbBnaJRM4iANbHmu/BC/wU98cLwSrUiyjpnmxkcYTwBSKnm9jm2VKTVXWEzN6AHAPNau5GPxfF8tlweS/vP6zL9eSU5Yp//iL8t8W4lO6sXdT6SY+u/LVvytf962abp0d3jj7Yyycfjz7+sRs/iLAwATdFH6F3kbnt3IcTgF66DYcj452W7MndXxLNgBYBqZUnNZudXKrVCSTJc8T2utGjBkzomEc+e5xI/BpxHG63SOOg+ex4xrY53ZO3a8CvpB+NmiNam4M9pdjZ1XBTSto+RGoRRJ67gl8pjmREXXAtqPTlhFW+KTnfgvfSI+muZUMIDWULtbe6L6LgoM1eTjozGahkQUGjQ5cAyLVZQpZlrerzUhZmmZRRyDpNoxMo+jcY3GHErrRGCdkRTnZgtybHg382tImjRALJkdVFJeDSAmnlGSBXE02o8NFDHTqpZUOwdQ4cy18TSKu432IoWqixhKOWAZMtRHRNe9ZMn7fs6TjmT3KT5/fp+x/eurjpHrX4yT6xG7lT48/rvzxsffeeO2y+3XNrtmX33Udsd42w9usu/qBl97jH/gJmbj3aeWFnzyjvPTc06Rj3+PK+489Br/4BInvhOfO138XeXZ5dvdzyqPj1vy87gDVi6u5GcKTwhzOwFnp5FlWcCXUT6vJrFM//PDUE8nJp/75z6eSJyeRxcpaZS1ZrD5wfXcdcP12GOj60DPHtXMf96VoO3pxuWwhzogahS/zNfj9TNYSBZEaMQpEKgPGOJUpjG7Cl5DMN+j60HZCP9q2AOXSbFJqdCafbinm2lporq0FWSNKc21A2EKSVSySfSjeARQ/Ls0ormvEaXMchBrZJlU58w0jUrT015QFVdYATCCNprSvp7RvPzbaJ7WuLxcEk8KArq/hOYDYyIwdD86fd2DOqx/Nv1TZtfDm41ecNfeyCcOywK8bLltx0sUh5VUxRb6jvC+mUiIvKdKMhWeeALTcoTuHz2n1cq5/vVxXrJfLOo6OAVP95Mp6yI5/O6A8oDsnRhxJbQ/DFmUm74DIwg3eFSa76WIID5jVg7IdonAayghOUZ3w1gb161qzxaH8Ld6Z3nGjT5neMNd51tLNF8xsVzqN5rGNze2G+2eb551w9oVe5N/ZwL+bNf5tdQlZAo4v+zQbufZkRSYnABeTWUp+BtlANig7Jik7ig+s51SZJyTAX8E5x//g8gHUNFE2MwpRlw3spI0qa5tYnHLUcnJ6Vh4+gQ4ZdYmGsC1V8LEpR19am3c0FD15ZK8BI4+8Q9JpI4/wRZfA61wpyexg4495+LLM1KOgM5rY1KP2RKf5wj7gS5sHjK1Br847YscKdmOiF4KBcxR8EL1aP5tAXFmXUR3Yw7jfkIiW1kvmPLz9OAudeLzgxotnJybdfZcyj9RNaclMQ/9qzsO/iFqJTsnjyOPdVzTvVFK6z5unT8uAK4b6nM7SGf4A8u/hqoacpvP2TtP506zltWSazku3WpWZpqPNGmTgTJ3zwL+1Kg+XGawzPINNHUeu4zf3PZ9zmPM5hjzfENN+dVQ+ys38TUSZKTf499+qGGnne5/2GQeHPJ+/93xaYF5yPj/NZpXDnwsRmNSXGUuc9PsDb49RvlC+0JdBo/5HMSIqn8WeeoqhsnjWPJy1lmvA3RKDnrVOOysE6JjNx7DXR3eI+LCWGU8XnOzFOM0xxh0odCOwu6RQy2x0XUaqpb22hSDzsnGjV62DRQoBsaNCJ5grbXanKcxWhZSgok7EJWiyLwyfo+VQArwf4WkIBPzfBFEQuOOugeiZbC+MOvWE46OpUeYWfJx+fCxa5zWScsja//dULpWZ/Df4WNWYObISMKZT8bWW8l6EG4HZzeG5DzsH6rMFL3PL4hm6ZcXF5obUfuwk+GO4LcXvoo0qtjYadWD6I+ks6MyVJrE/SrhBubeSDOLSlePnPw9088pxNzmnv+un4WJLERebBsWFFKd+dylK6tOsX4RiolDNdHE17WEu+FlchvhIVosAv+AMedFQ+8W8GXNofTARYnIs1wsYpvjbhsdJaZGlHE6+v2bOnDX475QZzaNOnJ7NTi+LEgv7oTVzWqZNzWVxYJzOIRv2QkySAHn6gG1qwRgfhy1D6tCNHI4DWkhoiInLaE0IR8yiRhy+DGnDl6UTlyE6cVmXliIHpWRGG7qMUHsX0WFgF8EQ5RvNXdaXn7vM85EYhoE1UQyHcSbNYoIfqlXxPHAaM0unMelkGuDeFRPdN/MbdwofToxd/LtNxB/p6IgoH+8l/CnHTXQead2mLH7tgXPJ+YDiXc4BM5olCH+E2fyZ6mzxKO5HQ08X47rB6t4h4+bSKUasKqTM4KIe27wxrhyMgx3vMFcKoj+UrKtvwh16Rmx6MbmRHf3Ap12hZD3drWdsAqxxttq2Y51EJuUcqaHHk8lf+vtZQw0s93ynjA9GZ4LBjga5GPfLf9lUcPx/aCoYC5BO8IXykWrKjt9mQphQz2PwOeGeEeCBlB8WZi4I1x9vv/6/Am/VmWNDXSjyT6AOPaIhUHeB6hiVwR36HdQ30nD3PuCuhnv1X4a75P8Qz9X28lxCQ1zeHI2rM8zfCHnUYxscf0de1jy3wXHIfDfqtzE8ruWwCzDL9fyLMImIbM4WqpnjkgLHpWVQzHaFPLwJ1CR8qwyOu1xikw1MjJVa/ib4NBTec7jKtB5DIteg7CrrLepSqyExP4grNAQhjpT1iMqpBF9/n4jnRuNMNegGPSdyp3J5A50I0tNsYp/JZidWj0pnmUU6/a2NMxvMdFYcu1/sJtXR4bQ2d1IyZT0aQ6Y8G7VmKurQaX1GrrFPAj6sgDNVwplOG2LS21lm0ls22DP9Z73pwgpsG7CJbW39pr6ZmlBnv5WLenUCO05RE+C5nPBhO+gCE3hNJ6v7thA/rkzexquzmQ42mlJ5EDOWeWel1smAg5qVNNCoZHO8rJmhkjaFqQkOKnFiyeGcmoix4/WKVH+cCbRfeAXIF+ZVE9h7UZwWxZ5hPJ2DnhQzrrFixjUIp62hk6Ie3NLrsYFAcFZajTPAC1X0hQi4+jipZwMPviCYLQ507eUIqJmC0+sL0qVgfdOyrhIYhHIp2koG0PMavxY585CzFDDt5XZW78mp+0tErpq7UZ0RcAh0BJk+27X9JcF+W2o4YjVYU1KI1RwibJm5g7ZGA5Wwgc5J67nOCC4mLM4UyDoICLTBAtx4EqO76bAOZ/axOmO2ZOdJApNrxb0nuD0wl+/kx5G14BHuVfJKRWfnC1c89o+/KG8qrz1CrlQ28qtn83Ugpj9SnlMeV36sywX5ZE+3ifhJmsRITc+fZ/fOlW6DmNcFMcubw05fS7E01l8wsg3G6M4LgEqqTWM4hy/W0jHzWlq6Th3TnLYbFN0IFhKPcGCtG8OfOAt/hp7hxohohJvNcEdE2Rtr6zvFLQdj8M2aQaa5ywbJ/Ua855cPkMtPfvcLkVFu6Bw46BrsjQthj2eZSfBwuUnwiNod12XifQEa6w4/DM7U4WAj4S+Aahx6LJx/S/Xp/l8+N1OSg52bzEKtOfTJya+LelQ7+/v07NWDnD1a7uyxkrMHjxnnqmId7PhXa0p2eAhK/BkGQ57C0Ih9pANgAMVaiDCRi9TQMDcGEtWQLvjZiw00AmzwmbX5aA1aH5aNmIA10pmtQpx9Fe/FBA5Jx7BaZLK5/XxNX1zIEdzPXzcETsoJ1mD4ubK8hA2Drff7SRrWaRjO1lKc4bbOleWwBoQfmcX1fx5aJdMWd/biRg6Dyg47aJDQAI8NvVjB5Z24+7QLsMIfK38M4nANho7HBnpbwwjt6IFFSOSfX+ov018EPkUAcLFEtddhDRNVgranEzwK2QXAu9gOWDBDOAuD2zldaIjsWOaPint0FhvvRZmQzAAwbY+sAkw8xRnNNlHniWv3CUR9ELF7KRrqjInWWpfbR2oRDdRwtdQh2MKDay/bvZtC3rP9yjW7O1+fyL9ywZ8+fnP1ivc/+n0HgnvJ3a/m4QcA4Au3viqR+T1bhUWblI+O/PEmBPyLLdRu0Vlz0FFVXATnG4acNq/+Z6fNo+q0ed7tD9GE27FMnLP4uszcOXHhaOZgw+fklWJs/X8/jEmqvMvBuAf19qBA/rQYAzMY3wcYY8PCGP9nYUz0whg+ZhhbNSVfBsyPihWGISBVNbugwrqW7ioZVdyxVh5aBLYpWwgxXVXPbgUZDnrclVYNaouljuWUlSb1hsZJRk0sy0b3sWJkMLVWBkH/GKjRBuf8hgHKjGdz+yADbtBl5w4/uR/EyX3Jk+k3vI/zSyXz+yF1fh8wyRbSDz3Dz7i83CQ/eaPI5uXm+TU2x/lrdd+OCzyq9SVxp7NP3JkPqVPYBYebM1tpCKJNh4gQKpncmUyJO01Xj/X3qKvYZifWN+pQ8wMhN60s0Q1qffdQebJsCY8HfGKtej/9wQd5S+dtdAlPZ+eyXTsflM/VYqcu3MPzXDBH5j8q775X6/MWntY/wjVzD6pbyLG/vdig30SnR7XxoWYzmKHm4pmb6cBQwcZaWdRstNuMUwmSGdd50sx0GnvB6WhQfTPwZwinKW1ilyFci7lnKeXM+6LYnoU3CmC7JoukpECb3BTFnT5md5AaKYSaMaxNoM3vpLf5nW5Ea6mtoyu3W7Ee4hHdt+7penZ34zkzm08+a+6c++9u3xRvjPlOzt576hmnnX7t1XPPf7HRL7z/3Au/69ZbayYdd8JFHePvXbZpc8Czf0GodtepV4xt23LeBTd5Hr/7yOHdrD+IzuHr76D50hR3lzqJXzPsJH5Dn0n8kdRvibNJ/HjpJD74cXW9k/iNmKvH3vIQ6DupTixUOgKRGBtV7fJVVUepI9OgjuXXDT+Wz3IuwwznX455mN8MOaGvX0eL2D8smdMvxU0ScHPrt9xSMPIYtxQ0qlsK9gBOammtQnKLgJSGERQp32ZXAeskGXpjwTLUGMOtLeA/7u1D0fCyleaJm7j7VbzUDYuXkX3wwnzdJMNLshQvSQdOlGh4GaUNl1CeSYoQQksjKJYi1QmGpbyvKk715kgVSSOOgXNUKzrcZodpmkmtG5KBdJcVy/clTKThajHgKsON475QcZUbFlfHleAKA6kEC6QSDXSIEScXm4ohVxNtoG4KYXQ1nmI1S7HalcliE3CIITaULmTYU7YU2VmHPKaIbNRydVhlU7sCatOFOtYO0A5UGAOSW0CBbSiiPUHRfpyK9jGYLHOY2V61RAN8HjkcGcr2BgxDkvGD5EGGJtBNfVsGtpWIOsRrjE42mrNv5sZzL6mUGjEspdKllEqmpbHFnHwOHIB2So96Ro9QPadm4etLSVDvQOEvIcEYNDQZ+LFMGpDeLU9AtQmqQa5EQ5MR8w7zCLQuY5zqIpO0SoDGEgIMg/jSFGYR36UZ+fK4X6E5UOM0bDdojtQgeH9e9aKObNIwLmR6+xQY3l+hOhbx/vNvpWVBmcg5CKjHpuUGNO3tQ+ncWrDvGdbTkKEaWNaPyVDcF0ay5gbEd6ZWw3dMLJgjjjRl+pHOMqtjiiw/Nge/M7K27dgV9CCND0Or7MiAFojh9Ldw84B2CKab9E7hZW4ENxowv4/LJzHb2pBF5EuNGbalwp1FzEtjM6XIb/Un3VZQJFm5VcCZOA3jKYbxVCnGU6wdGLg7AN9tpZPDY3DhiIUxNs565J11acrQYt4doY6TzynH4ojoCN6aF8f2ki4uVpdGIrTSixkR6bLNWR7ZPsRrmMQ8TLnQiTsN9a1NZDwB5CdjA3H9+NR5O34s9/wHnz1/3ui7R2V2nPbvZ299ATB/2fVmJ6Cet5FlZ5x1zhl9ET5j1m9/NXv+1avGOivHRBuvvx4QP1lcOOohwLz8w4fvRn97ljJT3ZVDvSzcllOI6bgHdWWnLUHj45Sw5jyoe3Nw93Yt4LC2zwod9BvCFtH5lFDh8UXjCcqntXRhi5EtpwigJk61SR5wJ6KJWuTYihiOxzrbyi0WLF2wU7aLtNzWnf/q3/EwYA2Psn9AmwPwIN1fAz6WBzRAnLuk/wYbcNwLIbbBJlRME9ENNiE6rRnuvfNCNqGT4PXRNSd7BIuryk9zRHpn2WU2oSGX2ahdkIOstHkIHcrCIHttdHehK9nzK9xuUwpfCOC7YLgNPYlBNvTUqBt6EK5wdZwtBO2yO6LsmppvvqeHuYaDbuu5gnqFZVf2kDtLfUEG31a6BzSJm1v7wpcA+KoZfNUIX20RvmoKX1SFr06lXx5ioDaW5aMUDDMKAqSBUBlIq4cmoubiDUbHom/XOggthY2aV8foqVPhtQG8Ma6ea8G+ur4Q1wLEcQYxrnutTkuZLKbB0SEYCYoyV8RAPNPlNaFDEMJ91CzJC7joSusTJjXdm5bTYP5HY/YX95e5BGxbktJs9ynuzUIkxMWhkFBq5xkeSo38QJxcoBn4GSpOgkX7PgA7/0c17T1PIF+815vz5Tkv3VODfe92bo5Wa9fRPEkl1Xc4CeighXYrRODqaD/mgtSBf4NZWx1OZ9ZM6mR1Bd6WZW3rV24Hy+pFmeyiu2jYmrnDD/duomG1Rdydk4AzmdmZSnfn9K6+cAy944Qty6FnwhU5lBDlFuSwRCRbjUOFqWQtTm9TP3/0LfiwEOTHxDnxTKzOjsOSmbyNsOluulOx8iAYPPCEaJHdgf0glQ6twg4frWZ1p6JaZJetFVRORjUXG5JdxcPVa2wfpOfD8zAm74My4eghOFuSzi8ht6/V6uw6elcK3c3lwHnztFRdLLL7WVq0d57Jpi1XsRWXq6jrQixmdWlIl9dlY1JARSCEw9h+ulbFYaXt9QDEwIJ7v3K7QWPbZgpT14CxqMPrSiiwu0+tnd2rtpMLcLXYTUCn1SK0qIUjFria0Zqm/Zy9Y4roNXot7GKXIIdVGLPFiroqJu4VjHq7s6oCv/I6ZZHanpoI23hPrS3l5i6jXXSqYSqa3to6fa61ti7n9SU9Rq+zmpTZOLJy3qjNytt/Xrpqz02k4sX7dur4kg0kN1xOnnv/k0T+xs6F62fUzBp55o4Fl12lbDiaUEZ9+qc3ntz7yi+6djN46U4YiElDXJb7z2PeCiONSBdqWMBZMwIpWFNnRscbox18MU3HltNhjEJbht0f0xUO8BCbqhNiwy2T6aq3ueCn69Re9XShnvk+2IvkykJUaqiormF5kt4NMxVsVYZcMwJw3zjcphlSLhItt35GOVw+/Bx8LY3u0v4FQu0uvzu4Cnqr2ilqddAt9PoFWk2b3t7JPAGDuqtPRE+nEofWfWLBZBUcNgp6wF1ua42WIyuzu+aHqDMfH7DARnc99WCu7t1jU3peb8l5+27WCZfbrKPV4AsmwedHGy4ZRDkQPLYtO8xLGbhr507qnvRfuEN2aopVO+9WOK+Lq+bmq+f1Cb0baLS6u7pyJqCeFyXbjZ5IJduFGEAM2wQnPTmrQXJsJKgMpos5pTLIXqlpXfsAhAuri25GL9I1GBYDDDGukduhwpDUYEiBbqoqTnY46foWJyhYnAGxshfD1JaGK3vr7nFwMGJxTAtVMmGqTBdixZI7qDMsVgAesLs6rm4or49j8d3qrAoLlM/wTncKveysUsdgSrFQPqVTBiNLygtSYCB+bu4jQD3LNSTpVBzZinX2S1UsBYTeCALXQYzM4kKevnV2iEC7Kn3ogeGFQOoNmQB7V4MhDC8m4MVEGh0xVmwXQWkLpkC0957MgRzQJ6vSC3Kpy1UE/37NZAWLAM8o+lpF0PdrPtZ8jcs/K62tTz3aKTwtKGpt/VJ1w5S2YAlr63RDBo0shIOyy1Ksrlst2GXIqus4nUGczO822UUDq65bnbK5krI7W54nVYl7SKVVoBV2XBuAizN9rexOSVprRyTUGdU6u0HdL1s3dd2at96957tzlq1drUzfumjuKWFyT/vV61dPH7fqxuuq5+SvIdynRzqufHLdl4o4di2ffvgsftKcfT2/nvlv7971HbBXdP8N6B7UPBuOcQNO+Ng34KCGEnCdAGZ2cPO+y0+DDcni/IbrcJiqHbAU5zhQs+UX4/B6VlzvC+N1/xoYcctPl0tVwhaMHgOssehYwWN1xwHgERG1cXkAv+6NFTUYt9JOqFuOEcboscMYU+mY91VVtxUpGWGUBGiD4W8EbbHOPgDgWk2TDwqz1jwlqDDbVO30+DFBXbYvaAAWunyCx8Q6hMriA6/9pFf99tNjaLtwZjVBF5U2aDzO0aXFDq+2k78sfw9SaB+AoXkDy+zlcUUml6mx071EIAuYMTnmzUQAYc033kzEOHqo/UQkqDL3YFuKvu6tqdtw1x/dBVrD3c4x6uLdL7iHVo6zTTI+Gj0VV4Em8UZ3ucbcLfN4a0oN7ZOrwSjJT/vo/DTk9HuBnDVsGBHcK3Y5TA3ON9nobjj0RlyovEI4lizFcXFR7yJaf79FtCXL/tQuXmaljDaSNvRZ97fmsTNXXH3VldesnkUXgY+5o8+6v3SVYc9hnfjamy89zfo0Gd1sQDfMjGw+NsphSiiTxdnwkgxJ+CDlWj6TD9O1FGGOrepswGWcakYEm99wsUkDEDqIWRFn3mgy0HrUN+KBwZh6KK7YVJa/B2ORgUyuzfaO4uzgn3q56UNMqrqLY5mAO9FEN1hiU8WAgVNv2XFkXFo1YNS2hl/VU26WW//SypVHVtDxY612qfsTFwR93YC9FLQyEx+2MlNXWpnB2Wk89AjqckVZgSBaWiCI0u4grdybKi33yslombX9dWrtJXns1S5EwzBVxZMBJx8OXULcBNi5p7RuqNHxZaBjmItyW4egY6RIx2Aat1Ljckj1dkYH7pvO0E1IjoMls0/YHVvVe0ujCGydt9togtQn7tUJZlOQtkxIVSjxAxii7DB2VttGxZLBA1gjPMNxyZw5F4sz9PfeeNO9leUG/l8+fvr049ffc//hj9nMP6vhrQE+aeCyWEeifDJyWD5pLuUTCOqjDCnROrqIEkdkE2yqCRhnBLCGJQwKZESfal4Wu/TTRe7pqnGnIbRJMAwm0oWa3pmlBAZ1Dl8d1RDNKgulj52F+iJuGGY6pw8WJw/de7JZxecRZz/eUlYW702u4VYMMcENMmbo7rNSIUlHtwN0Fwu1FwFMq1XQ64e5MOaSA+gkyZY4DhHrvW3DLF2ACAZB97DS0GAD6ySeTq30VU87ftyYyYsvuqLcZPah23PLR8aWNcw42dvRdBnTM8pK4RH1LpBWbu8xVoDBTsijAOp+7TZjjq3dRqrFNodGhp82bL1BnYOKRmrExoYGqnGc8ogcvaF+FGCptjGDL0bEvLsBy++0BpznqmJtx17wHYjGYe4TSfZB6HB13p7f9EUuz+57Vudh1x/bjc/xb3rjc0K98bkguP2haupkf6s7n9XRi6FvfqbBxlDj1LoFpTEHvaNEjavOH+6WkvAgt5RE1FtK9gh2T/8k1je7sISBWPbakigrufWrmmJfkeZfMlqy+dxjpGXym9KytpSW0X+OlmwUZehJ+YuLEynDkLQ4j6LRlMWRlw1H0+ggNMVoEXd99kaLSF01XjTg9Q3flLbF9F858l6owVmWwhpwGo230z6krcdCYyAxZuBdqPvqS6kdonsRujGtdyyEb1B7jLpwi0INIzdbn4AXk1LKD0Px0oGSoWW4Q7srYkii/0G7ZaGX5jNpHH1tCc0DPFss5dINsrY6UUp/OlJjoyM1JaxQo0bFXRV2j8BAp30T31C0S3N85QR8ngb1QBb4r15Q/zfkPPbPyHlWn8x6hMQwcr5yDLEQy5i3DwxFcv1FIAUx5bN+Mn7JPyvjss9flPBApFqV8OC3kHAV2HL0XaVBOIC8+htVsHA3AM71Am2tAFkUb4il07y80A0BAHsm2mRvtO9kLw0P1NurfUy8cU4XAwMfHfi20wRA3uF1URc3ytPJ/GLZS53cLcIgFMd38c7nReQyskR5XPmBEn9SA+TnVzz2ZbfyuvKxermwskcpKJKyS2d4ngF0SDaROBlF/CSOsM04+onepL+d3rE1intyqFu28hGsMFaz1YPsxi26wqB52Eu3kMTgcBSaKrgsNsQ20YbYiDnVxSXq1fmRAXdy0Rs7bAPu5qqGZ1N9U1vbsPdzJXPtfK6lic9pQj74hV38W8efUhtOhFtTmUlbBr2965IJ9hGj6kR/qK66tmHWxJWb/n888413sx3R30b9/y2q919d6v1Trqmhd7NJzixetis5MngLmE1X7CkKYU+Y2s5Zydo5cQ+BO1ZcoBGjCzRitGEihg0T2r2AshAoTS/UoFsPsWFftqCbYXMxj3otWLnr25S/EhdwxKZNg13idknVnB1VjR/MOfxpv4vctN0ro2iu5bf/qo0hWi4m+j+0giWmLqXKmyPVA9euHOMSEMDpEAs/XuVXDbJz5aWVK7X+M92fOA/nB316af9urIjQXQiwbqyAwBKwpm5t5TQ2YQVoG1pQbUOL92kjDIrFC/GwflDlp2ZSa7sKDNl2RfNug7Sdncav6vlssI6z769c2VPAXjONJ5gv+Od/3RaZUvdxCLYo8SePnUPQq3TG/6eXxQyx6egxzckaZC1MY9HT0noXZ9LezHruqhLeiaJXWVviVfbv02zo16eJleMEICfR27I5QhvQtrgC6FXSi7zUYovGQ17xGDo1h2zXU/s1l/b6luV6NnlnqS89Vlmpu4bekdzIffVN+QqH32oNx77uqWk4piok2MJg0OgJh7rQFdfd2LOIzUE5CxsTEtgUaXPRRJbJSdc/pZgekqtqaaarFhdrJerb2M0J1bi61TP8Sij9gCzN4BynXNA/5VWG7fYNyHmdoqws3tvZxN2p1gWjOu7SEpaTRqblOkNfzksP5LxCDcNgTS/zYaGvBpivYBECLpoi9olybCRVZQW7o04dKnPVAX7CNZjsko1R1ll7jDzZD0GDcub3+yawyrLnkSv74afvLrLf/H9nF1l1okbbRdZljsbiajT5TZbgMe98KJOo+eeDqLjntWhjYH/26n+2P1v2VhW7s/3hCOvOLmAnwbdoz9YCkcHs5IkaoBcN1mq/jYHKmrO12Fm/EfgmyjVwO9kNDTR21gLmAt4whbIEmsyf6TJGaPg8ojR8xj7aoLU7H6TsEkQOU+s0XuAcQ9Bt6uWCtBpo07IV9rPtqRQisZo6Rve8KVlLYzGvSPukZGNETaX0D67FPsG1tlEKY87STNoicvoXHx4fnPLC+Ue5v37x2ZynR3WSz9avOnQHDbEvKkmlLSTfGaf8cVdzTnlN+ZnytvJafYhcExg9OqDMrG0hC3WLenNOOnZnLfBIHURt47kTsIIztv+ttdNKbq1twVtrJ44t3lo7Q7u1tl27tbaZKaR2hzxZvbW21T1ZvbX2RHil3YzJVGOgvmEUYmqy2GFhF9eOSKVbxqp31+adI5so9sa2lL+/dqJ6f+20b35/bTFXJ3zri2wXF/v5vvmNtvpHi9vN3ip3ua3Gxy8DHzfhLsZj4mMwF4Ukq6ElRyLDJnELT32x303lb3XLzuhiy1+hgX3V8C15f5TaI1ioFMTISKoHigzv+xYM36fSNhzX396n0jY86+uvVKtt6n0KlPfXAO/nuEncLOyH7ujP+zNLeL8NeX9qh8r7UjaNN5MixkdmEU0j67EnGjB+siYTx2syMZrKRFfD8WbAYT3DYX1aw/3xDvkEgr+K0lIYRV/sGuc+QRWaUwDFx/de9Syf0IALby12XyQ5MgvcQ+dx6WXPHW3lhWWqKiwzv4Ww9Ct9fluJWdmHVOFvLjaGZ1XaHZowUGo0mdkOMjMS9w8fm8ywrnYa/zT2kxK5Dh7rvqVQNOHvx2g+PSL+8zLRJ7k+pETw00qy68PZgQ964wJNFmaqd5fPxFt5S28vn4xB0QwWFA17k/lJQ9xkPhFQMw1QO63cpeaz+l1qDko+w5T8Xks4mRrTTq3DWFGO2Nv+V244L43HvtVt52dp9PnG156Tt/rWCZSZ+o1AryjXzHWW4XepOV3e3cmUMjpWb5squlkmleZQpVhGavqWbI87UZoS2CktRirLs/2xsPvoPslW43Asn70iPiLemspOXHz18EzfM6Em5/N6YvHakScdN+/8ZAnvv6by/qm4ezLT3w4Mxe+zh+N3wPC0iYjhaTMBw5Mz5bn/tEG5P59M4Y4o4Pw9oPXHWNqZk/S/wf4Q8YErz/8TInD24qs1in1zIfgj0KyEhL17Qzer8eLzaqeMi3XKOD30xk29utMK4sO8ke60KpiraJhoVhtmeHbrgymT99OpJX9QXWnkLw0M/Q5cZ4ut83iNm+gtLj+yZfJeuvzI66b7gmlU6Oe1zSVesStSHcf1hXiZYZSuB62iDVu4th8XIcmCo610FZIYK26WpQv7xZLrRbBm4+/k19Ddsj1/6ezZ1Lmn75LqXKDnb7xdWy+bC/K5nldKrxlheHNDkPiR0ciFuVqs2ATUfatBXXfe6qhCvGH3KvarJYCZtbbVurQUOij72A32Pnpbgg/1Q4huAgvRptUQ3jziY4NAWMKxUNxYbPBjokXb/EUH6HwhusWbzraKYpcpGkuq7ffxGjonFMSO7VBbSU9ruF9Pq1i8WF3DVrF9200yxulPTWATXLsWrF+nLF7jL+5JW9ioLF/W1DCuOLS1oJPObOFN9bcVUcX461Z6d0QdJ6v8FWT8FcA7IzJ5M/KXmJVN8FKlw0VZjk5werLAdEK37KsFlquJUparQZaj+VbUGTFrt2ypgV+IUb6LJeiG6TwfK/KdKSPF6Dpa2ZbMZCibuoFN6eIXP159F6E8Su+VcGQw7cpB7I7Mpd25ISZy2RzbsBVTP6ulDI21YrnY2M4f/rCXr370I5Wz4OueTbmgsHL2+8hP77OPs4/c1ctX/Pt/YLu3kJ/0t3FJ7gfMImHBphq4SXSHECWoSAtVjJviaUmflRMGLGYVWas2LUUPSjUZOYhN0Qb4lWCU2h89MlUQr6eOUisUpVwWxYW/QQe9y5UOyHqKA7Ieh3ZDIE1feILquG4VbpdzRGl1kBZIMVmP/w/OTIg74CPiUv46gJc2bQIm+mDOueW5SOecs4POPsCHLYZRnMDZIcrI69BW69neOGQc5BqpkmHBxKaQDQfZ0jhDcWkcsISBZrMMOnY3rMguCcRaQ8kW6iC/qudZdYv2SytXHhrfb+f40T/Dh3m6P8FZrLhznJ1FR3Ng9DZdk3pHLR7ERg9SCZitpAfBSeO8oVI7BXw0AX7tyG8mvXpjrtB7IDhOCI7zBhs0Xr1y5eHNpVPZeE/8Sv5ckKtK0ECbBt1/LgXTstcAx4mUXYNe1X8NOt7/ZUEj7GINftg47+JZzt0syg7aueVFhvBjl3jegt3zbRJh+fm+W9MHJoO1DepLy2R+GZYH5ntTAKcbfDUzrebs7Dt9Xo2edZJ51sVxW39adhs0LYED6VI8UzKTjrvO+4+lI9gVCLbIfA+jQw7Z8IYjWocRAQF5a3VSzZDb/DRD7oYXK0SvmiHHIrmk6z/NPhAFpU6wOuV+WR9shDUnt5fcfZCiP6q5smwH/rkQt+E+90WD84AnXbCrkVoV1ZxgYoETsDzeywzUXvfZiO/XNuLbPQM34peAoVH15r4VJHU7e2/diNA9Ak6hE2g5c7AtAhXHtEWgEsWmzPYAhtESQaF7SYVx4N+4QU7UW+Blg6E7b6Dj0gYeFwKU3mEAgkIOFnjmnvM0dV16owFKsXqdAa5YQPkw8Tge5vHRlRt2UQoBdxhcbGy9gt1j36qtJdT2b6KrWEylpSZFXFNH3NrJ67bNfnzpdcdpNrbii82+wPPCp3S35os3/XK1/oU+PsjRl5WZ/CL9Vk7kAtxyjg6fy2bAqD2NdjOvo+fUEez0CdJrEwnr6SVlx8D+n96uBrit6kq/+57+LEuW3tO/JUuWZNmxHUe2FMc2xsQB19CQZvlJUygQXEJCBgib0rSwZUtKIYSUFggsUNoAKZtl25R2JFmh5WdKgKbDwLCUoYFt2TaU0mGz7FCWMt2BRK97z7nvT9KTJQ/MeiaxLcm27jnfvffce875PshTYg2D5EOglzx+DKVdMNmpuVkv4/CIHTtAGJeoiBGwNpLJbDSZmzz52yR/752nnOtXB/L61tj13xIOUZ8MPfu8ZCPGur1JehYr0XUswg1oXQAs/ixKYTX6dCnRZzuUbhcGsiopfYhFnSFeDaTo/3YATMi4pilFP330JBGDEnkuBtd7nTlkno/B6cobwCk9gDzzCKxiW6B6PTOoyBOYAb6ak5YSV35jJ9n/PXkFP7VH/lnlLwcP9ekHLC26vIW/Fj7L05UU3f8f3iNPRkflW+vOVjzXJ6+xcNgXNcDdzWFZBwTiIRoVuDx+iAqwV9WKkwjv4pVYgFon/CrQR0CY6QtrDBJhBH0YJ1IYeqN8C5rGx7pX+0ABmWJ6AE+nYYzEOVFrjzJYZnSMqDt/rX36yAi54RKy5svyB+S8zfKO9bJ83aXyjmuO6gai4cCJcfJgdGgoJL9beTdEowBy9y75fRZUfnhajYVYrmuLoun4bWVFabPV57rcaq5LFQUJ1u6GCJqqjbDNW3QDnyk7nYeUtbDkFQMwJ9xiSfL54asOFAEvtrEkVcEiVrOyisncmM8gNMSf/BJxHLn0lFXSiZeqdES2+ElI3mGQEqHj66f/DdttyLtydtWKWbBkwe1NWVdgHVcpV4ouz8RCpCv9+/aR2e/LuxjpipVnlVfWI3odNZ2rli10rnq4bu6fW7d3oTur1uA1M7kEFaUUjBJoMHJSCKNyqDo1+AJqTdxdqhQ7Wp/rZiSxzbxgMmc1f1R+WD9VFa/UT07qG9CUpXOzja69X6r2TWdWc0+0qXtorFEIQbepy0kDyWwxREfvguWLc4lMLycGmx3XifK0pt6rGZXix3v14ejeNFlm1HmEfk2BOntTvxa68irPMbDZlL1Yjoglwy34mJ44yt2sCDGAJ5QAcNlwAXrsnXd3Bxyao6G82B1gfMcd3WLVRCtaDKwCZvOtpuywytXyo4ZKwxpfC3fVFRiijU6nwWjMehv1t4H3mUUvSPrSzkoymeOxKa2rFd/DCQCS4WAND/K8eJxgDY+fNZ9zRSeeRoGPxdIO91u4KRv9L9SO9XQEwIxhjDoAhLdM6id1/0eBLaPVeV32ReBMXvYpwWWsJfd7gQGIBtfBqkkNdUDuYNWk1jwNDViCY8LMzdW1TQYPb6kLRpl7+d9p8ShbX3+KczgA6grGOexDKhCf5lF1z2jmTuBo93Ycgw2DK/p017lrKrWqSZjYlM3pob/qLsPbZfEz+inE7VI4593US/hFgHopgPfNAbF2t6NBfxBrWt0sXHLj1ZsbvBPEY3EQjkD+XMFdpQ0N8b8I79s5UZDEgg/IsQLYLkvX1yKnFLqqDskErIpTYPcffBKdkZdfVBxy/E+PkIfntvLXYQT0l9KGUuUVdMholPxCuf/gs9QXIqhmYdmzB0/Wmv0ltTKXnqnhsKLQeAEmFXIvC94lAiESi1aLdofhPYpj2oWg/yA196FD35cj++ibuox/PTxUmd62YVvlNQg1yHO6ntLvOR+Nvm5tqqdEYNFvQSVpYSUkqKaEppGSzR/C2IJNBIMWkrkGkk+NLoyiR7fr4UWdxNGP9T2dcVoxzoD9LXNaqWv9QkxVTcmpcIHvhMjERudIwSfOOxWKxg71nlcnpbIwUqomZFQCWsKMfWqJziVnSjVl4JbDM8l3kOO/B/KhzbS0erLqLXgz94OcW5K+JggXlFwwyqZdshku4D4cs0E2uhHalY2QA/2e+EQL6KiPeow4ubA27KlFi3nMkJXXCH8QDlPcDKkxQytcaENZtbKxGXTgaD1ATZIAzjQu0QP8TNDF3ARTWSOmigMJ1prjY0cYyxDYqqNvojmQaoxmBqmVxgirEbDMYke2thxFDYlW1pauT2BtietrS2Qxa4tgKKwzwmZTdTFdLWgMxXTqGrOHjhZvt1teY3o/gTWmT8HDvM3ehQ1MsMx092TYMlNKphgT+2IXGq3e0JTp7kPVOI1xYd2m9zYxzath5OLqhFvuesUroK5wHFPucWolz2IGybNIy5JnyUAj+aqb+O1NlLvexB4C5BCzvEnfdwd93xco71tSGcTC9H23Z6FmUn3fHtexgoflHIAyDt63R6PkC4hFhzCBZHGs5SgsmZPFASWFCSvatfz2ynP1FGi3bt1a2azznlVr4qW4T5sqy6XNzNyjmPkgqOJ1dS9CWy5Akguq4z2y95D8Oumj//9HaxJ56mZlqeFDTGleqGZETJsxIvbUMCL+FBkRlS3Ypm7BjcgRYTyCkaqtmnLwARhSUnPHGTC2Kt4+GIDG2qczELIavo9sj1pnuDiN0F/kSlEcjyo2n6IDU8opwnSqW7Bmw36skGGcDsU+4VjZ5YsphA4ln0tdCkoun5pInbfHDOWriQ5UWgJz+N2YTfKj02kwWfayi2BvTXMo8MiUI3oFaw/sMzGsJYOKDeCDcECve0Gt7S1yCbWAw9pAWt6q4AYao1L8OgqUM4nIX2UmMF9ZJ78jPyE/+vJrb6+dzl349mv8PnIuOdteV9JMPkvOplj7L6fl7nb5HYUb0boD9+1+bhjq5/FWsTOPTaJBVfMHkAOEGX2gOJ8rD7QHwZ4DACXWadjl0vT1oH10IAfthWCJNgqsnNLBQbHHWgmtUPw8SDcaT1AR94E7Zq44QE9885wUxqvzPiOhulr4HKxioVeqK6bIMvqAzcrqn9c++9xzGy/+BTmv8odbdudvHczuW/vbL2z8t6efKV+y9YuXWy+/8qov8HPk78/93NwmAOAtt0Ap9HM/Qsr5G2+svP/Goef+PfpS4Yk9hb0/4FQdauu5dI2A2u8euHmFfsxCe77oo1uyE/imcKWIU9QFlK8567FyR1DiqZU6lKoIP0ohQ7rJ41fVkEt+j1pVX+BysBYCfabVzaipOumhhR47E8kexlNecim7UwcgqM0Pqo2SLnDFIhO8kVbz0T5V+WmMHQn40068d+cN5Io98lPyrv95oXLkwaeL8/fe/cSPtil6VweevPnZ3sp/Rkf5NbIFstKWByo/2/nAfTc8b4iH4Zz8MF1jQLvrK4Zzsk/ZuDtYJae6cZc6UKCsw60kHugeLtH9WeJVneMSLxkvKGEbkHggo7UHgkxutEO7yKvbjCdJIN2XsunRfv811xCp7eQXT/YMnpTbnfv+b/VI/8u5C+WrosHKqcYAX/XvUfRvH/dAI//21fo3oft3iebfxML+TSv+hUSix682AqTFg25HpFNtBCi5orEaR/c1cbTWBKB63MTXa9Uw5DLmdHN/k1+xSOTEh+B3TcMaubW7uO0Gbu1YruRTAjXwt8qzXQrhwhoKgr/jSLkdpf6O4qkcCw/ao0ZRa4jso3CX5YCkMQRfTkyAF9wi8gti/6wyOjO3S+qwnq3xvBpTqXTcBvcjjnfTvfIDOq+HuLzGD5BS6/6cNo0UAJmT6GiBYb0cZ2u+m0nxBdvoq4IZh7oXIEOSnYOqfDGybDivUJyM5GBcqQjbFeziPCcGcZFzAgm7iXZECirDbIqImciyhXAEUbbV3e8deejnq9Kb77n53qmNO3bu2Dj17htX/vazq9KZg+d96/6pjdfvvH7jFLkIKff+uu/RQfnY/tVb51atn1qanbzgxrlnjvQmyYqD2TNuv/6MDauGlo2v38H7q/hR+Ffo/F7CLeNuUXi8YRfoxsxRN3jQlD4Aq+ML0VzZzVTf3Ez1LWAUn4r3TgDT/nSbQxDD3Zn+QRR6C0jzduvSIcYIy7hwkUogUEslEFCNA/WYusZbECTesmSZEl2t++Edl30vOzB69dmf/czauz6/Ml388ewPh4dGzrz2mfvO33Rh7jfIffP4P96eeHFr9qyZibWduWWnrV/5rTu6pLdvnHn4q8u7kjMbyTvGGIRh5SjFyooFsTK2CKyMG7EyslzBSi7/SWFFy8O2BJcrNTbO1hGj521XI2ZgjRgBhbck2Kef2qcfa6/6e9uU2k+foMVg3jZWQtsGZUrlIMNMkIUN8TaQw8Eq2CCUcTqYrs0gBQ4vuER7Z7IfgROnB9gOtmr0M1lweqArtsVZnhq5yLQOlZYAtFq1wrImGNLZ/uULFgYSw4+8RvjAuobi51TuIYaf8riiW6NAqDCdLY7oVCSnmSNpOTXJFLXOVANQQaPQ8rgoTbc7KaxSA8vGJk6CvoZiZkpkMjaRERplLRs7aQoetU9DMCoizTvCS2odXmaaNi1B7c1akZsWIXfcVy99w+q3+VdoDLuEm4KaQlivyvnGmkCT2eJSTdyqermC26JxatxxfeVaqaxc8/HefrBYISuCab108VqaWz46CQ+NS0WxB9c1CPMDUtFuRRW9SWrxKaB7nu/PjaKomzMPVGeB6EStYlDThc3M2M3WufNr7dxszTsxXm/g/4e1LzeqrH355Z/c2qeEDC0B8ptqCNEiEIWM3hvK9su36Nq3hK5931zEfplrvF/mDahDCUt9y1w6PMK2zJLdOsQYaj7ejqkSDzUBk0NjHmoCIv6E4a4vI6+xPG2domeFfu4xhYulA2s0Uee3KNowv9aXheuheYubOGiknFeO4mU/A5Ef86JFp5vaz6MqHJc8TrUEq+D0FnogaU7jTChx7KGfnN6ihR7KBf1QDuF2KRBHRkKnWA4nupPQX1q09GBozRXd1I7zgTget4oWaOl2ehKMvURMYpfFoKodDOSExGxKZix7Z6Lx568dASHhfOWiKMnXTkILTyblwy/b249fgRH3vf6XPzpgNu/8dF0bstyAWme72alE1TrDs5WD2o7D9mK4OuVRdE7XOwu8WgjlQF+XbqSlQJtG+t2mCaCh5FlbgJ60+M5uJkbZJZYkPGQWeim+XIwGWOqkdlEUz2BDXUnyWPOsETqlekdF0wXK/7VNK197bXL4pHPTN43PXWypW/fPG+o7cmRdZceqU72rwomrL+PvOr7OdI3PcJzlaaVu79sNcWQAUHxRAArkagCTUM9noS62N4ahnrMU6WQHMwvc8zk9DbGhzao6PBxWp5EJDPgnNd4u0F4+3fqv1Pf93PcUBijoNjdxPMFSRGIB2XCsCsfSA8QDzqPAqyoK2gJaHr4aEEAf0ceqDrDuazoWalOqv9gtToCdv2MiKgVyRYdLpXo1R0Mdz1MNEsjNhnIEExT8S33VhaopP8yFWc3FQirrEToPHExK5eNJy8d1afnoIqTlkwET0fT3+O0NRdKPbN2qc/tb3sSb/5tbY7k35AJapfiP1Ug1AI89cvtH2H1LSyT2dJB1hPV5fnsDgnoPDFD14WHqw16usLAPe7LQNQY3C13I4d4VoePpzmHXTDOn6pwSkBLqpt8l2XfJZg6HRhoQJSnZ/T0tO7y61drE9SRc1bPbEAbkNa2rWtV5uIZioQfq61rBQipbjjKbRVN43QIlc0BkllHBUQjnqvFRDjHLhLLmYJmPu+w0bkvoDB1xQI4n1bLaQZVx6iCztsoyDfQ9RL3ZXGAYskcohnq4YYj4F0JRKltcIqBYfUy5rm4FOxmma5vB0yiMHb5LNMMOXHSDqM+8355iPB4aeIrLltCvEpGFYdRAxtYMUNvqxGsbg2quVrVW1dN4QdHTeLxVPQ2QZQQ2nU6FQqeBmgYwh4PFzAHVhYDqZy/pp7OcqQWjOJCAohpFO0Cssxs7MeuFNYpLQVS4K9SixIbRlHXwe6jOjg0WsO/WS/+qWLT9nmIRcgO7FsQizsKPtx31KtvRQWCIZnkBUSxCxNbaQpUxZB/NUJXRM48N4fRUdc5RqNLcScHtT0uKNOnWt6seXZEmxrpuHvVFOqNd8RTrbyx5vImJiUVq0qhmqEPEWboNFhTjUQxA3Sxvse6lZ5wwN8CVGALKMcZAZbIgLclCqx4W5beyEnWzFqTuZsiACv3uCAQq9hg74ZS6MkvQKmm69BR6UWk+0g1tS0UOOkXt3vBEs+Wopl3JDDK/rupXaoiaj86t46aKylstRHiE4mYJN9/aCtSbLSZtjDeuRfAo21eImTGULcdZKxvQyoVAr8sKjHpxO0hCMH0MFz1HIuuZLwmQi+NxyCrOu5jOEy5EC4Gr3mz1Uk9Hq4xmDrPjb1RZTOBCHGe9y3YdPQMBM8f9XMmLTAWs+xGR5lCQxjRi6BFoIA9NoqAI05NDYg5J6YaUNJ1YCWs+JSi6jrolxyDSeAWyQFkwn8QH+tzHkIYDmLnmBT4MWapiMoqnZvokFJrYFdVITtTTVNAEwpRilINRG2kgERPix8rkBnLVAVK+S56Fk5LcTwbrtWEQXJW7+SuwajQlT6MkwuP8W6S/Tv6I4usSemaE2g6Jzky1Jr/ddqzUjuXL7VAZ5ICzowBXWJwDrrAi2YL9VeWOqmTDyimbsw1T/F46GRmFGSZpvTY6br7dF2RrMPIH2IGHhOA9lROTlARFDETgsKQBkNb4cQn/3kf/e1HiU8KeSjtJbf/zyORstHIAkhIbLLv9r5Cp9ce3BtsJJ99qaADhuTG62K6j6yzUoH5VGY9LOFZy4XigP4D1nQpwN4fnO8EB90ydIA8MFBMgZWjH06IdWvzsWn0vkDxDtaloh0G5/MgNohT5AvEzHVSxTVDaUnFYNhBnYONifGswrjHe8rvxLf5Eln+0coL4Jq7vzGcjcgATsELW59310ciJI1H/YfnJqvpK5qfvUj/Ftd7VBf1UiGXLQaXKPNHYY1AJFuk4BkwDBucBFZvPBhO/nU58bwRiWX6i1ofm3jNg1tSHBbVku9aPhr5H1Y9rUFHzu+Z+DPGMMwM6WU192t3Yp8A30kmH3VnlXtTjhAjB5kI1TjjOM465RfnZGFaZertfq1qv8fjRKs4V5vOj1OfRVudmrKW52WWYm+FFzk3lzsbMtSdeV+9t6lz7c+NdMPPtHqzl3LmIOdrVwJ+RXM00xcJNB/WjP8BoHnhXEJU0JGkx01UZqqkPK6u1esVqH2pFisZ81incAZOcQOGUrJoWWGmeFoBwf5KGM8OTUNA7PEoHuyxXmGyQL5imDw4D619iAMlbyk4xkkKRgYyS1TqFjnccczCQOpD01MFi81q1DVStJdtDevVvi0mEj9436fHUc1kTai6rhVxCYSJLCifVpxPg7m4FtfHgCrDx4DB9ZX+usELPM0xW5eUHxYMsswV2W4HZ1JRYFB0ThqRWeILaOQ/clZhykJSUw6JyWbWsRE0yD88bWImaJbBCJjbVcxAZ7r7GOYhM1nB93Gt+fQzcAaY3yPT/mHaPHHJChW7NjTJUHodgciYg1ZDBL5reI9disT69cJtuHLPEwm4Te+h5hT7uOy3fLWN+oS+LtU4tXClH2um5jUIP9Re5SByl2bDsSb1M7hGLLrxMlvroA0mkEfFMFBwLXizXYqfmXnnEMAtNcgv5GmsQTrJcrNQ2DXIFDohUyryD64DIog0hYO3g7NR7Tugi4jml21IrKpDUcgG9HECpF7GM8gnrC8jFMMuBNINHwLBagJuTMP5qF7sBcWGfKyx0n1barCQXlH0JYJuAp7bctu5KY3f9TRBnclcBY91Ax3op9fsWruDOlu1M3NqOBwA7bKVSFkaPMtisOo+DvSmE75YJWUOBBrWFonoNda9FZxBq5d3Ix+3GkKLIa9VqZiLWkrledb28O33Pa8g9/ByfpacfDo4Ua3g/uee++z7Wc397nxywdPBeeA4oVjosMXJg82btOYfxOYf+HHdYmCV/ta7jvFxh+qsWWzt9jSTaOoSOH1wgWO4/X7B0S+TPEvmVRA5J5HaJzEnU5mTLNomslIhXIkclUpDIDomcZXhFFp+irxt/SSL7ql+UkMh7EnlKInfg6xL4OseGi/Dji/Bx9dWG767Wv+NWDg6KXF6UyIQYyo8Mq2xjwZACoMN77y7Ee0+bGZ06R3h712NDa8e3Bbr86VUrxj+HYy0LM+RP1nPoWKn9MK3Xq026ciGemYGffFCY+cz4Nn+3Lz29YuJzZ+5+jNmpJB8mf+RegZ8N1fyovaT97J3vVv3ogBIXCrPCGNo4SeamP0QrB0IxQeTEVNofuP98v18IJRJGq0+mSU+a+NPknTR5PU1eSpNCmuxPk39Kk5vS5II0mUmTaJq0p8nlx9PkrTT5dZqU02R7mlyaJpP4nCVNPkiTF9LkR/hj9PHl+EuPq7/0F/gj9Kkr0mSd4Tee9Gt8TvtTZ+Jv1N7OC/hj9L1cZ/jJdvy97H3Qv/d1w1sZTJNEmnjShL/4Iv3ji+rH1erHhvpnGz2nICEfzubFfHYQEAGQMEdFpvaBSxSYLJ8+i+zXv66BzAWz1d9SP64XZoRBxE/SBEHW2gfWa7AgEvty+u/q0DWgf3vhLIKN/h35sDCIWEuaoC1T+4D+d+4k+7U/VAvF6r8zAJj+EinxnxeAW4czrsNfum3T5ttu27zpNv5h+Ez/0ZXjN3/7o72PO4Jrf5rjxhrcehi//s1sPj/7qeXLP8X/pP6rC/Ozs/mRM84YqfkM72snt5rPCOs4G1iZRva+9E6y9py33jqH/ORUcql8nXwdm5OG1435hDwJkJ3worVyaTXZSXbKe0+V98LrEn/7b+EPNK4fht+Xp5YbS4/mJY6+0QRBnsmUzZ4MJClGQvQTxsxMwirF9SUu5/nLtyUI98rqey+zzPiHvnLyHfdIZIP8AM+TS+SH/A9+/TM39EZOs/yg/Cu5kiB/PWdUmg2dTdyED8RP7BzMHn3j7JnQuHjOm6+PDz0ov7uSdJwAH8/xEfuN2H+5lK3mrF/CECWxsmAfDRjUnl6C3Wpz/OF+79L7u3bI/7AjcWdO5N/fsoV0fEAc8ubN8v4r5Zdj8h+3Wq755S+/5qys9w4OevlHXF+rPEQ/F7z9lcf40/u9lbPw8bnKJv7+ygPeQWojuPh/RHgE9ogRGrTB1yfW/R+MGzfMeNpjYGRgYGC0esg7O/dTPL/NVwZ5DgYQuPD06SsY/f/oPzcOPnZJIJeDgQkkCgC13g87AHjaY2BkYGC/9TebgYGD4f/R/zc4+BiAIsiA0RoAmVcGQQAAAHjajZMxaBRREIbndje3qSyOxXAch1gc8Vg1lcpxhRJCIkdYRJZDAoqEIJJgIUFCKguREDDYSrBKGVKFVFZaxFTWUSxjaSGWNvH7X97Ksd6JCz//vJ15M2/+Ny/4bjPGFxTYta/hjD2PzFrgXnXXZqsdyypPrY9vAUzzfyl6YY+Iz1jPwxtBx4z/GTgAC2ARXAJrygM2vH+a2B2wrBwej8MtW44f2JOxI0vG+tYFOXY3OgGrrI/O1tTrhU1LC1/cdD7n/xPXd3s28bdYK3Yi3rIAviKb/4vkeaMzu3wfrBnZ6Q/sNc5xmzMarLP24Dv8v+l7SNiTBp3TQ+xJ7DbapNhTvreL2kN8mzPm+BPWDeWjbgDXQJ2cN8Jje1fZtm1xlFun0B6/tF8HgepWE1f3mXSWDb8Fr/l3lzy502sI0EU65k6/AbD3unoGNeUn9lqh3wikJUygrTRKpNUwUFecSbtBoMd5r99n8MtrVmhXxqzXbhB1cE6sHl2tMtOzao9k9XrZeupbdyVNdLZR7OaKu/8Hd6VFwWjaoLdvTmP16lkzq7nxcZPuLpifQQ5v2Zxbv3frVPPj46eGsOpnBXPfuiv1qTtue67pHWgWPfdKa81Hy80pHK64eprnxn+we0ea5b+4lNvP2qY4fGUW13lfnoP7ZpWP4OoZ7ARegZfce5gvwPv8OQ6wD8EB2KeW6vWF4JOtjpt90V58LwXl5awX4mN7GO6Z/QYKROv8AAB42mNgYNCBwiqGDYxTmDyY7jCXME9iPsXCxGLEUsKyguUUyy9WCdY01gVsXGzL2L6xV7F/4dDiWMGpxJnGeYjzGucPLh6uTdxt3Gd4fHgO8IrxLuKT4lvEr8Afxb9JQEggR+CeoJvgDME/QgVCr4TThA+JaIjEiewRFRBNET0g+kaMS8xMLEQsR2yCOJ94kfgJCROJaRJ/JDdJ6Uh1SH2RjpKeJOMis0eWRbZB9pOcn9wpeTX5Dvl9CgwKQQpLFFkUkxRrFG8paQBhkdIb5QYVJhUXlX+qcqo6aixqWmpxag3qOuo16meACnZpOmie0qrSVtDeoxOl80/3lJ6IXoLeOn0F/TkGXgbLDFkM04wYjNqMfhhnGd8xCTJ5Yxpgus8sxOyReZ4Fi8UNyylWQdYc1lNsWGxKbN7YNtieshOxS7K7Zu/nIOCQ4XDAUclxi5OJ0zJnCecuFxaXeS6/XKvcBNyS3L64l3koeUzzdPE84qXltc5bxnuFj5zPAl8e3yo/Kb9p/hL+WwKUAi4F+gUJBC0IdgneExIWqhB6JWxXeFSEUcSNyB1RIVEvomtiOGL8YqbEcsROi30XpxE3Le5HfFuCWEJbwrnEkMRdST5Ju5LNkvuS/6S4pOxIdUq9klaU9iq9Kv1ahkHGjEymzA1ZTlmnso2yD+Xo5ezKdcm9kpeSL5R/qCCo4EFhXRFLUU0xW3FF8YuSulKu0jVlAWWXyoPK31W0VcpVtlV+qsqqelMdUX2mpqJWqXZVnUFdTN2yegYckK9epl6r3qLerX5P/a36Hw13GgUaXRqjGicA4ZLGHY07mpiavJrCABnq3TkAAQAAATsAUgAFAAAAAAACAAEAAgAWAAABAAFgAAAAAHjapVVNTxNRFL2FUgWVFTGGuBhdKbEFSkiMcWMADQaIESKJcTPt9EumM7Uz0NSFaxf+BuPKX8HCJerexMSla5fGpeeed1umILowkzc9792vcz/eVERm5KeMSy4/KSIplsM5mcLO4TGZlleGx2VB3hjOy6wcGp6QK/LFcAG2Pwyfk/fyy/B5mcu9NTwpV3NHhqfGDnPfDV+Qu/l3hi/K6/w3w5fk2URgeFp2Jj4YPpLLhRnDH2WhMGf4k0jhqeHPMjXAX8dlthDIqrSkgZVivZSaBOJh+dj7QFWJpSN96VKriVNPbuD0Jn7LyH8Rq2iojLMH0I+hGcKTJyvAXdjr22eEWCIpyQ5QBJzIY+g1ZB/6PnQ2KY+h2YeVeliXNiQN6jewL2IdWx8j74SnJ9h1ce4iemBXAr+lM2xbfGu+KZkGsG7Tzx7OYqmfymwdtlX4VKuINVP9Pn4rlHfJRv2mZOLq2iKXKk+0vm7/HKy71A3wrg7rlMD/vysyYLLNSAfU2WS1dZ/QPptrD7KEPa6xPg1IHLsK+f/Nj2e2PrH6VK8H1lu1uWW51/lOMDEDngmzbrGW2dha8yY9DPrQhjSlbhXnIZ6+TWIbNXCxKtaHHie3aVPRpl9PtvDb40zErFZ07Tp7la2DdrVuU+LRtgMcM4uA8gjTliCuZlIjU0U+b0cFFiFjO25NTovP7tWsmykzSDIzokyVdYcnRVlj52N21tV0F3O28UeProJpxpv2JCTfJOM7IttgmKOrtmqFFsllHHKe94b9qfOWuooG9FY8o+Z11ia1qDEZBXhcx91sxbDdZz8icnbfgPRU5XzWNza7Dm9valzavAFNTmBH7sg8nh6fEmSj96I1citKxvx/bOflEZkFyKPKbLdZuwN2VTOdxzw5D6sjd6kLzSYse5xf7dqG2UTcae/22UPX10Hn7rHSVduN2ui9OvlNK4Prwhk5+hlfJVazAWk44lMz2MD3YwVzuAXOa/ySq89dSCvDPruvpztV7g9tVss4V9ltxC7LMt5L0Br8LywzquN6f+hpW14g9xYkOi3hbzqdRMN42m3SV2xbVRzH8e+vda5bu+nee+8RO4mTdLuJs5ombVJ3pPPGubHdOnZxfEtbRkECIabghWfGEyD2kkCCFxB7if0AD0wxn9nF8T2qXcSV7vmcc6T///zPYAyl79ISBvmfT1uK/xiNZSw+qrDwM47xBAgygWomMonJTGEq05jODGYyi9nMYS7zmM8CFrKIxSxhKctYzgpWsorVrGEt61jPBjayiRpChKmljnoiNNBIE5vZwla2sZ0d7CTKLpppIUYrbbTTQSe76WIP3fSwl3300sd+4hzgIIc4TD9HOMoxjnOCk9jy8SA3cTP3che385CquI07+ZKHeYBHeYPXeIwBEtxdPIW3cHidN3mPt3mHd/mBIT7kfT7gcZLcwyd8xMek+IlfuJVTpDnNMBmy3EeOqzhDnhFcCpzlan7kHBc4zzVcx7Xcz0Wu5wZu5Gd+5QVZ8mucxiugIH/zjyaoWhM1iUtCkzVFUyVN03TN0EzN0mzN0VzN03x+43ct0EIt0mIt0VIt03Kt0Eqt0mr+4FOt0Vqt03pt0EZtUo1CCqtWdXzF16pXRA1qVJM2a4u2apu28wRPaod2KqpdalaLYmrlT/7iG75Vm9rVoU7tVpf2qFs92qt96lWf9vOi4jqggzrEd3yvw+rXER3lM77gcx3TcZ3QSdkaUEKDcjSkpFJK6xRP8TTP8Tyv8AzP8iq38IhO8xIvK6Nh7lDWSmbOn0mF/G42XVNT0+IZrTG2+KPDdiKfy/ptTys6kHfOOpZdwh/NJXNZ57Tf9gw2J9L5hDs8lHHOBRPlfqB5MFewEwknWwgkLnetloQ9mnLQo6WY3y74Y2ZBxzMQK4c6l7v+mFnY8bRiXg6nRLCtooxkRRlt5VzJcq7RrYbCYWNtsL0iOlXu+9oH7LwvVWz8HabGtLHDVJM2x9BZkeFUue+tUBsxNlhddsItOFamhJndZWy2urw9ZUr4uooF+zLFxur2orIVUXX1xojV7UVlS/h7TIU5z+qelJtN2nl3OGO7hepc5cjq9fLmPXq9PHmPPm9ypESwr2J/I//dX8ScZKTW2u8FF7xa4qYW1zyluPeU3BJV8Xw6m6xyR9vq+BVVupUjf9yctWtuvt+r7EKJQH/5hi9cecPhUKOxyRg1lk48XHz9xpAxbKw11hnrjRFjg7HR2GSMeoZM3lAoMJROunln0B5JeVPhVs/6Vl/MzedKg/rW5n8BNmipUQAAAHja28H4v3UDYy+D9waOgIiNjIx9kRvd2LQjFDcIRHpvEAkCMhoiZTewacdEMGxgVnDdwKztsoFVwXUTswOTNpjDAuSwqkM5bCCZ/VAOO5DDVgTlcAA57NYQDuMGTqhJXEBRTmEm7Y3MbmVALreC6y4Gzvr/DHARHqAC7gA4lxfI5dGGc/mAXF45GDdyg4g2ABjbO40AAVTANWoAAA==) format('woff'); + font-weight: normal; + font-style: normal; + +} + + +@font-face { + font-family: 'Roboto'; + font-style: normal; + font-weight: 300; + src: + local('Roboto Light'), + url(data:application/x-font-woff;charset=utf-8;base64,d09GRgABAAAAAEScABMAAAAAdFQAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABqAAAABwAAAAcXzC5yUdERUYAAAHEAAAAHgAAACAAzgAER1BPUwAAAeQAAAVxAAANIkezYOlHU1VCAAAHWAAAACwAAAAwuP+4/k9TLzIAAAeEAAAAVgAAAGC3ouDrY21hcAAAB9wAAAG+AAACioYHy/VjdnQgAAAJnAAAADQAAAA0CnAOGGZwZ20AAAnQAAABsQAAAmVTtC+nZ2FzcAAAC4QAAAAIAAAACAAAABBnbHlmAAALjAAAMaIAAFTUMXgLR2hlYWQAAD0wAAAAMQAAADYBsFYkaGhlYQAAPWQAAAAfAAAAJA7cBhlobXR4AAA9hAAAAeEAAAKEbjk+b2xvY2EAAD9oAAABNgAAAUQwY0cibWF4cAAAQKAAAAAgAAAAIAG+AZluYW1lAABAwAAAAZAAAANoT6qDDHBvc3QAAEJQAAABjAAAAktoPRGfcHJlcAAAQ9wAAAC2AAABI0qzIoZ3ZWJmAABElAAAAAYAAAAGVU1R3QAAAAEAAAAAzD2izwAAAADE8BEuAAAAAM4DBct42mNgZGBg4ANiCQYQYGJgBMIFQMwC5jEAAAsqANMAAHjapZZ5bNRFFMff79dtd7u03UNsORWwKYhWGwFLsRBiGuSKkdIDsBg0kRCVGq6GcpSEFINKghzlMDFBVBITNRpDJEGCBlBBRSEQIQYJyLHd/pA78a99fn6zy3ZbykJxXr7zm3nz5s2b7xy/EUtE/FIiY8SuGDe5SvLeeHlhvfQRD3pRFbc9tWy9/ur8evG5JQOP2Hxt8ds7xLJrjO1AmYxUyiyZLQtlpayRmOWx/FbQGmSVWM9aVdZs6z1rk/WZFbU9dtgutIeCsVivND1dsWSG9JAMKZOeMkrCUi756MI6AN0g3Se1ellm6GlqOXpBxuoNmYXGlgn6D/qo9JOA5ksIFOoBKY79K6V4qtC/ZJy2yXNgPJgIKkEVqMbPNHpO14jUgXr6LcK+gbbFoBEsoX0pWE55Bd8W/G8BW9WNboZ+b/KPyWslDy5K9biU6TkZpY6U6ymiLdUv0Vyi9jvt1boT+x9lTmyXzNUhaHKIcqyEaDkLfw8YTQBNDpo2NHmsVjZtrl2u/kZLmDlHaT0BJ1HTZ45+gbdfTSznJVOK4WQkWAAWgiYQQB/EVzAxYhheIvASgZcIvETgJGK8NfDdgN1GsAlsBllYO1g7WDtYO1g7WDrMcAK+a2UA6xci+kp0i0EjWA4s2nMZO6DNrE4zDDbDYDMMNptIHSJ1iNQhUodI3R4DafGzG8JSKEUyRB6VJ+RJGSbDZQSrWsb+KJfR7OAJ8rxUM/Z0xq6Tl6Re3iTyjUS9WezsQ+7e9L7j24G//uznFl2th/WAOrqPNelG0hq5z6Srk6Ub4Kau0Mv6qe7W7ZQPsxIhPcgeX3sPns6DCDjYSX/9rj3/7ka8bbeNGQXHE/UzyZb3Naqtt/W+FAepZ1J3mVOWPoW7ipYzFE8hSiE3Erfcabyo/I+kF7TVzPBMiq6VU3Wr/FGy9F2y1MD5aLfeG7ukh3SKztOQHtOldxmvgTW/3uWKBeLrqifdSuxbPeNypiOTPb/StfqBbgBrYCOIKkifoH6ou3S//oxFky4jLzLWvTSoV/RrU96pR/UY36Mdx9VzerNDbA+b/M8UzXE97TKTYCcvdY079Fxl8v2duY3vJb3Y3lvbjK+QWdMjScujKb226ze6V0+AH9gHId3G3ghxPk5yZs+m2BVzo4j+otuYZ3wX5ibGa4uP3R5tYufcaU32pGm7er+ninU2ffVaVz47Mt+tHXstTVvae0Cv3PeYTjqG4n5v927ukWDyTnDucuZXdXEerpqzcsc10D9M3nKnmNPFnZ6n7nOlY/RxrdBhYDA7yovKyx/Mq5N0vr6l67EIaA4ne4k5369QP6Kvpd4r8RRjZ+hP4PPkPrp4i832qOJ/AP1E1+ke7uE9nPDWJJ+Jrx4Cu92zEZtr6m93h6H2O7CDtjENA6eSpZOdzwL/84C8m3g93kuyeVN44C/L1LyIT7J5D3gNqz0SVjloc7lZuAc7/RfC3NHu/+dBU8tP6vORAnN/90poeoM+5H3vIaYsM3omo/oYwfVdgLgpk6+vWxvGSuQWfkuMV4v5+Q1TAaIMIr2ZVYhyIWLzCipijKGIT4qRPvIU4uNFNJz8aaQvL6NSeBqJ+HkjlcHUKCRHnkEKeDGVw9dopJdUIBkyTsbD80TEIy/IFKKoRLJkKpIpVYhHahCvTEPyeGVNJ7oXkX68tuooz0SCvLrqiXCezCeSBbz//bIIyZAGxCOLpRGfS2QpHpYhPlmOZEkT4pcVSJ6sk/XM1325WdKC5JsXnCVbZCtlG75djiSFI9uwkwE37hv6Md6G2cx+NJYVzKs3MxtPlJOQ/sxtqjzEO7FaBpk5PMIMZtKznvgGm/hKiKsJPjcw3oj/AIgWgIQAAAB42mNgZGBg4GLQYdBjYHJx8wlh4MtJLMljkGBgAYoz/P8PJBAsIAAAnsoHa3jaY2BmvsGow8DKwMI6i9WYgYFRHkIzX2RIY2JgYABhCHjAwPQ/gEEhGshUAPHd8/PTgRTvAwa2tH9pDAwcSUzBCgyM8/0ZGRhYrFg3gNUxAQCExA4aAAB42mNgYGBmgGAZBkYgycDYAuQxgvksjBlAOozBgYGVQQzI4mWoY1jAsJhhKcNKhtUM6xi2MOxg2M1wkOEkw1mGywzXGG4x3GF4yPCS4S3DZ4ZvDL8Y/jAGMhYyHWO6xXRHgUtBREFKQU5BTUFfwUohXmGNotIDhv//QTYCzVUAmrsIaO4KoLlriTA3gLEAai6DgoCChIIM2FxLJHMZ/3/9//j/of8H/x/4v+//3v97/m//v+X/pv9r/y/7v/j/vP9z/s/8P+P/lP+9/7v+t/5v/t/wv/6/zn++v7v+Lv+77EHzg7oH1Q+qHhQ/yH6Q9MDu/qf7tQoLIOFDC8DIxgA3nJEJSDChKwBGEQsrGzsHJxc3Dy8fv4CgkLCIqJi4hKSUtIysnLyCopKyiqqauoamlraOrp6+gaGRsYmpmbmFpZW1ja2dvYOjk7OLq5u7h6eXt4+vn39AYFBwSGhYeERkVHRMbFx8QiLIlnyGopJSiIVlQFwOYlQwMFQyVDEwVDMwJKeABLLS52enQZ2ViumVjNyZSWDGxEnTpk+eAmbOmz0HRE2dASTyGBgKgFQhEBcDcUMTkGjMARIAqVuf0QAAAAAEOgWvAGYAqABiAGUAZwBoAGkAagBrAHUApABcAHgAZQBsAHIAeAB8AHAAegBaAEQFEXjaXVG7TltBEN0NDwOBxNggOdoUs5mQxnuhBQnE1Y1iZDuF5QhpN3KRi3EBH0CBRA3arxmgoaRImwYhF0h8Qj4hEjNriKI0Ozuzc86ZM0vKkap36WvPU+ckkMLdBs02/U5ItbMA96Tr642MtIMHWmxm9Mp1+/4LBpvRlDtqAOU9bykPGU07gVq0p/7R/AqG+/wf8zsYtDTT9NQ6CekhBOabcUuD7xnNussP+oLV4WIwMKSYpuIuP6ZS/rc052rLsLWR0byDMxH5yTRAU2ttBJr+1CHV83EUS5DLprE2mJiy/iQTwYXJdFVTtcz42sFdsrPoYIMqzYEH2MNWeQweDg8mFNK3JMosDRH2YqvECBGTHAo55dzJ/qRA+UgSxrxJSjvjhrUGxpHXwKA2T7P/PJtNbW8dwvhZHMF3vxlLOvjIhtoYEWI7YimACURCRlX5hhrPvSwG5FL7z0CUgOXxj3+dCLTu2EQ8l7V1DjFWCHp+29zyy4q7VrnOi0J3b6pqqNIpzftezr7HA54eC8NBY8Gbz/v+SoH6PCyuNGgOBEN6N3r/orXqiKu8Fz6yJ9O/sVoAAAAAAQAB//8AD3jarXwHfBRl+v/7TtuWLbMlm54smwIJJLBLCKGJCOqJgIp6NBEiiUgNiCb0IgiIFU9FkKCABKXNbAIqcoAUC3Y9I6ioh5yaE8RT9CeQHf7P885sCgS4/+/zE7OZzO7O+z79+5QZwpG+hHBjxNsIT0wkX6WkoEfEJCScDKmS+FWPCM/BIVF5PC3i6YhJSmzoEaF4PiwH5KyAHOjLZWiZdIU2Vrzt7Ka+wvsELkmqCKHtRYVdt4BE4FyeSoX6iMiRPKqYCxShTiEh1eSsV7iQaqF5RBWp7FaE4o6dwoVhHy+H5apHH6iorqZf85805OM15wrd6edSAhGJjfSCa1KSp0jhWk4gFiFPMYeoEleg0DpVcNXXii6SBCcFl2qieaoVztjYGdUOS3XslExxjbAHX+fyZYFqoTQgdCfnvz6snaPcl/AK611DiLAGaEgm6fRmEkkCGiK++MRwOBwxARkRsy0OjmsJTTLZ82o4OSU10x9WiaO+xutPSM70h2pFgb3Fu9LS8S1RrK+RLFY7vEWVjAIlqU5NdNUrifomza76iMlszavpbRIsQI9LjYezPjjri8ezPg+c9blUG5yNc9WrAZqndEna2etfp3OJL8+6s9e3p514oCS5argkkwfWZa8SvsIiNZZEMxzEu2qs8TYPXqrG7ouDD7jYq8xevfiKn/Gzz8C3Eti34JrJseukxK6Tip+pSYt9Mh3P871dHI9EumTkQkpqWnr+Bf8pvZNABJ7CgCcAP2Eef8K+IB/wBfigB3+K4K1rqGuwVk/bDRoziHaDl3/9z2ByXjs1YMwA7S14uY92G6y9SVfeQV8bRZ/X2M8o7bo7tDK6En/gPKggqTzfkY9Kj5AO5CkSyQMJKm1BDub6SJ6IPM3LteRFZBCm4g2rKZb6iJyCp2W3BbQ0v0Bx1KnpoKIko05WOXe9ku5SZWB7bkj1guDahhSvSzXDicSQmuWsV/3uerUAxCOngyrHFSteucYmprTJ9BcrZrcSLCZqiii7txPq8CdkwVngQlHYGx8OdSnsnJ2TTws7dykClUyjThrsnB1sI/m88f406vNKJl+wMJ9W8uWHHvvblsd3fPT225vLtu3l+PLnH//bs0ve+PCtj5TS7afoc5L63KqKSQ9f3WfnS2vfcxw65Pr+gLhi96r7py7r3e+V6g1vOXb/3fYxWNCk8z+JC8WDxI7aDdzpTh7S+aN2ctRHBOCImuCor+2amSfY89SucCjb2KHsqKdKjwKF1KkOYIHDpXp13UWFzYDDfDjMd6md4bAtaGlP+O11yO4am5ACRlCsds6HP1Iz89LgD6J27SS71ZT04mI1QYaj1LRiZArwIRyKT6VeKdgmu4gxqCfVGeKhfpp1mfcnrZ43d/Vzc+ZXjbprxNDRJcOG3VXLvXVDtJjOgTeqVsMbo0v0N0qE/gPmbt06d8CcLVvmDJk1a8iAIXPmDGmQhakdzz26euCcrVvnDIy9NXD4jJnDCHiz4ed/El4DvrUhHUlPUkEiKegVMpBx2VJ9xIqM684Di3oxFgVBeYK6eXeCw04utSsc2kGT7C7VB4fxcr16FfxGPmy3ChnZHWRkks8OTHInprZjTOqeLbt3EJM9MbVDZ11rOne5ijJ1ATaAdjgp7QUeDdTEbwrmOGgjV4rgUzkmB/WAHhXBRxiPhj+x1HnzwMiqx18adtsa+lynLpP+0u81bumM2w7d9/Hpyk1rR2y7VisRTVzBtEEPXXW12q3TPSPLJtN7K98YYxvz4l+rNq+dOWzB1TO09OuUMfM+/+th8ZGBt9ZFZlVffw09JpqEzJEruEN9Hr1pYYeSroPGLgAbnCb0IceY387WvbbhsqkiXeCvkVGN3nmauSxb6EOt7+3XThK05Ye1TtxEaSiRiYdQxc0YbAWr87AveQpdpCidSpzsc7mBDdnkYRq/SUp64vDhJ5KkLdoJrqeTjud6l9C/3B39Vdvu1bZHfx1/7RiuM17brXWivza/Nl+n2puu3cUtF7q4nKJwPIHLE1PQ/fiRow8nSS/TeO3EZkmrKOPc9EYv/QvnK7u2JLpXe8qpPRx9bwzbdyo3m78B4oiD3EMgpIKzoQVUcbL9cyB7EczExZy5kp1EIQjnv0NUQvPfQfd+ovP+TPTqDoW4FMdeQaEuhdvLqZwjP58qDnSmVBU58Dc20BQeY6jE/IrIh/ksv+gx2WiOJzWD3iiMNdO+Aa3mm9vq3rvtiHBr6Uw6VVs2t/Re7YuraCft4560PWH77U+WC52EHRBlbyEKKVBMYZXa6hUxBMJD70is4DQpwUPKo6OEsGutY3EcdFwIRSxWfM9igo9ZLXhoJZZY5AW3D6EdXL0clPvTyHT6utZvOjetnH6i5ZdrafSYvofBmkadZBfoTBbuATXG2kxjQDJoUwKSKxY3qszgfhXj4Iv+6pe1E/p1OnHdOBe3Biy3DV5HpVI9/lBFKAAW59XyXtREwB7G3nyd6Ddct9JS/G41vHQk6+G77WIIxl7feICXQAny3nr2o18CsUv10vXr8ftp5x/g/s0wkEwAMiHwgVX1z/lpmKZxoyZEX5gtdTjzKcNMi8G3BA2f3I1EbLiQLMW8MTqVFN3vOpv8LjAi1fCwqk0oRlZ4ZJc7HHInUhcXbMN59PAi695x8ekjR/44feTw/1SqGzZsU6qrt3KFtB9NpCHtA+0H7XXte+0j2omavv799Dd0/Lf/+c+3QMeu82e4DWItyKI7iQjo7zjcEeVcGXsLEO8wsQjACidslkeBC9SiGzNoMxMRMjcLRL6L/rtSNN865Gw/sRvyaDJgLBloToKjiAMptgHFaCRqPF8fiWdXi09CLUvWAZPMABPYpSrBcpIHPyDZQdU8Eh56HLByCrzrSZTdEd5mLQamqDbgj+IsVuLliEQ8xSzIZBvO00T9oI6FNOYefcHJ4h+f7Dr2zGJtMsf93FBJjy6c+OzDGzZPFjw7Gg7vqPyfFVo3sXQEl/rUOyOWrH91JdIx9vxP/GmgIxe0JtIW6RCBDrEtbkkEZkRSkCQvkORlCMObYMmrtce1TYGQakfR5unuACID51L8iDcS4DihADEFnEKUgRBDyXIp6fiuDMdyAaKTiJzOMEscEN4ewYcfYgegjrYsdsQB4FBJVnGxYpeVNgBJ3GpienFL5JEHxsMOGPU5jYxhyCPYJnMsV/7Gs6u27nhp2bI161eueLimnBP/3L3/h3nTliw+d3CP9jNdJC1TXnj62SfL1sxesvbFxdLLx+p23729fc5rc/Z9fQR1ux/IuT/YgpU4yRASscS0qJbYLJwdgDoAZ6lekQAYuwoUS50SF0LlVvhQxMxciFkCJloYPLagN5FRuWyoXLRY4WTFwVSMhmVAkqBnkJjkmPpxax44frwi+h2XKoVpeV++oSGrVHuclpfyvbiJzD9sBZszw77SyX4SSW2UW2qj3FwoN4+tvsaR6jLn1fptqS4Qmd9WzxC8s64myUkceSoHcRxFlOSMAXPmyx1O9OVOh+7Lr9p8ZjH6clFxuhTXXjBixbN351UP/tkVztpqvA6PJy8CrxkPZTwUlEBli4nizacRl8erw2aqmtHTpxYrSaABbtRsB8g3QsxJxRfIFERpyvEgpO5Fi7q4fV5wBtlbufHVy9a+8MITDz8ZGH0ztz+6rkvRwik7jx/9uvYXOl168rkDO9cdHDrMxadOjp4JdeH58+TwUe3PdwjzTyuAV+nMVnPIXSSSgNxKi/knG19f685MQIjoFoE5bZk+J6OrCinJLmSK6gPmtIPfgWTQUMHkTmAampkGGupzAgS0uYE4c7EiyIoJqZE7E9BEvykfAI2UCgYKbo0RQoqak7mCpn3cf3lxenH5wLWf9dg55cDx3w+8o52r3Pv08m0vV03fHuBS6OQG2qtNRklGWsP78weO1H498rn2I23f8PGv/3pxW92cu5guDAAdRV2II51JxIwaik5bJWie9gLFXIfpaixFg8CnOlAHiRk2zRfr0cNKeVOwyE08A/jXT5zNtVXacqn5C/GGsjLtx+gebemMGXQq91dqIoglxwA/7cBPPwlCjnw/ifiQo8nAUQuu2wE4mhPwWYCjObiFjoyjCcBRCR1AJhwkuNQ04KcbDnPxXBwwuBOcyM0ENGnhfckBJ2MxMlx1E3ACObLq5OF3B7caJxXrULKoGZJkNi+AzTfnsKfZ8ZiqRfcuPvn3Xf956N5FL2hnP/hEi1bse27FgbefXnGg3ZYli7aqCxdvpgvm72nXVrl/10cfv36/2rbdnnkHPv3kwGNr1z360JYtXMH8Vavmz6l+HnVqKPjNfxk6BejIGot5LAJkAQcS0qw8cCBBatIpbz0qFIQ/JRBSTV5dp5LRFdhZymV18LpmyVb9XAK6BzUL9Yz4dKIJi5BeAkaRU5RGWQKBuJkzcLNO7FByftenmnb6i4Grr4vvu2jwhgOFNZPe+m3W5uULtmVtX/XIK/zuozRXO6md1QZHtfq09DEZKV9/uHzEGOr9cuOxRSUrP/zytG47GCSCQldWD+nQhCYYIEAsYUbSADshlAAvyBCFpRFR8PCzculSwBX83xBbcARhTo7QDWKyhXQiEROgalXCC1ljAEkxh7D8IeH1CljR4AK0ZMOXcYCY0pbGMJOwAq+u28IMfgn/EVydgFf1UZPPT30D+O7RlRMmcGX099F0xhztlxQpRTs9B/fzFN3Af85vYvQl6UjLqlNnZdQZxKCNUPh5iu/TsJvvQzeMG0dXjRunrzkL1nxHX7OokBYV5lBYeRZXOWFCdAk/YMYs6k4GL+CcqT04mvH0ZjCi65nupJFJJJKMPE2xx9CDrSV6SNfRg5uhB4CiSnIIzaU2zUu6C3lKXCOkYElsXBLoCh8PhuKRVYsLHW18CjpaKe4C8OCgviB42Bh4MAWRqzfzdRtq3l00o1dyBc29Y8JdS+bcD1GHtlkmlLy4+9DmxR9PLRwx6oG7byt/Ztq8h5fed279ypVAzwytu/S5+DAJk2vIFhJxYrXCElaLxHolLaR0KlBzHfXK1QWqD35lFqg8Aq++zCRyIOfO0X2sBMlEP70ydNW+s1P11KGnS+m1FzzLGSVpL6lJSu7ZC+swtPGIhZYcsCCVtgWaA3Jvi4WXM3PzOxV2w+KF5FZNbZAJzlz4TId88NVXFwE7EhINdrhJIIPwEsYYI/3s4mauO8xLzJ70D3AkAMd++EQGofobPWiRh/n3GW76Ga2gi+lS2Vr3wcB75MLnyh5Y4vGf2Dhyaj+OD1lvKnr0RZtbU7Sntb9rI2QPnUhvHlLbK733B3dqC7VRXLHr1lG3P9KZFmQM7PigQr+mGzlJS9WGHNb2lQ0fNfqXgxoNFxZx0X0LR515iy6i27R22jxtkdahfbB/u470Nzp11au3T4UMlsvwJ/0M8oCsXvgG4oEJMqH2us0qfJgFhVrJTCi4JQlxQFwBy21UipHAigVMAPdBPsB7AkAo124KlzXr6Wjp07u5G7WvJVE5exN9WhvHUcg9WBzYA+ssZvmhH9Ycb3gHJ3hBFn8y0Av62XLMCwaYyJ3o/kMAJJje2pz1NaLNYwYDgPMpYHagyG0o/slCKlH9TpYioi+ECJuhY3JIxJojvayA7uUDhbGDPfSl76JzJy7aEP2HNo/Oe+HV6jXaRDqoasurivaBqOzZW74hI+HQwv2flK557IGNpcsWP7RMt+WFENs2g22mkrGGZXqAHk8yg+jxgKsYaIgDPBwn4Lk4CxppGiPNBSS4WPVTsYQYDDaF1HQslrhA+4TkYqRClRJRIeM8cMqUoFeNXODVBUj9UZ+4VOp1o4KF/RLEM7KQ5v72I3V5uPKEd17d88MPe1495C/nPNrP3/+m1XGjT9J4OvqPb6Tte7XDP5z6t3Zk1+vSl+fonehnUD7vg3wsxEM6GtKxxqTjwdDsjdUiFKsLUQHzIz7dfcug+FgzCAB3SU/amSBXq6mNjtDWa79DutXxMPVrP36ufSQq2nNa/evaj1pVKc3/Yfdxms94iesPhfVt5DpjdUtsdQF0Q9RVUeSZKuJGYmk4S9EtgFQUa0jPx40kXE/A9Z89/FMNx7i/R6/hg6JSFj1aFl1fShrXHcXo7q2ve/GaJj3itLamsaDtggX38C801HEHoj1wsbfujt6ur7Uc9OUD0JcMrKmlxfSlFSWpTUhMQ5DJ8uFAK/qCkNMUisQzVYuHNIvZga46aaA6yTKzhwRQHCW5WI2DNNFAmy3Uxyfr6iODMchMg5bTwj9+ohYfNzlp364Dp7T3n3g3S5tNz3XSogc17XVuCMjUQW/9aZe0fLt2/Gvtt+PaVzd3pLPKomevm0mHNfG0nsnyKsOjmHSPoojhWivPuGptkqSN9UcUm15lFljDpFGG2IAJQ64DTK3ge1RUNBwQleit3OazN3FV0RJ9PUi+6M2sBhFoJsPG2gVcDX/ExiseqUT/pH/3FsBmKnzXg3rnaMyNHI25kYVdCpTfHctcWQ5k05Vfz1UcwGsL5CiKu3l+AithZpmTXdj5Fq5843OLNlee3PV+xVS6TKpat32F4Dl38q2fxpXtNcd49jPzjzGeWZp4xtsZz3j0jM7G8ggXwooaUXm7nlFQPaNACsE5+y0U4nQQ2PYW13MxF93ALeIejT7/NrCvhKsSo8XRgMhtiQ421jbB2mIsAuBKBg+lGA8jPNN6XrTEKphMOL49lRwY9dntTfYkdYRryeQ241qmuHAjJbGKJkvsdUaa9AKkKhPGSMUs13BinB0jskmv92F1JcLbHCwKM9ooaoQnhwapySPvWc35JS6xqsIqRb8bHD0u2WA7msiBhjzAzebOakIDjS6Jzm7SzVNMN6+9SDebKyRoo2Dszo7ixt1xLGszG1tSeUtsQ0WootQk76nku0ugowchAJ5Lo8I/z94kHKfnUsG/zgLb//7Cupc5VveyXLHuJdj0uhf4/5ivzSAeNF83+Fssgvlm0Y6UUIF20d7VGs4T7cPK+o8+O3nqHx/9iK4/kY7U1mo/nNS+19bTETTpZ+1bmn7q1AmaoX17QsfvyJu/sfqFh/Rp7g3B/9dabEwHLS1DgS2E0cCJBV4jGqgem9wy8AYDibQp1v7+r3Pn/qUtoHNqt9du1xaISv3efT9G13H7X1n28Gv6Pmadby86gFcesOebSURGXvljvEpDXrVhG/DCBrwuNcngVRBLE17Muh2yjbWjZEiMABXIumalyaBOzVjo5Ux+UxbDaZdg5MTSs4O1P7s/cP0lubleOzP4RP8zqakXs5Qju4CfH4nbALsHSamhbS5d29QgsDQxmbE0EVmayShKAoqSQ0qSnvmlM/SuiCE1C9UgSTfzOFmRgapEomMd5uqV4EVYB6BBvN8Hfp41jZqJYBc9+e+zD85YXJGRNSMrbcsqbSy9++CO7a9oD4nb3j847ZXcNtsWLu07oU1C5oJrFz24KjqJ+3PN4sdXge1gLl8JculAyluv/2GTUU2BUJYi47mUhJYdxvbNOoytNBTN7bGmZ5ODLK/FJmKNw5fVvtUWYmY45AdCfaaWLUQhKKG7HcNN0jZv+Sxy9NQf1HP4nw89yE/6UN12cMc3P/2ufXf0i7VVdIX08voVsyue6dZj77rqT2ZP3yqK0vJdz02b9GTXHu9Vb/2AThp3SEJ/0QFk+BjDx2C1UvN6icKHWEor1aHuR0RWmRUBFEQk1naVsILXlBFiL6CDUKLZKrFScnaHeAPzR9Ws14b+skjPhlTJ8L2KtdFd8lgkdOHFWPUD3SWkLljsZaVwiDONAQfLGtWVX6m1xyq0o//+QTtGP+O/bMja+e6h1/H3zw1R3Q8i7v+Q4Z6AUakkHBs1QKzDAI1KLLGiT5j6w0WI9zMW0B2pkJ9uXxD95xTwcdeOHi3shFBKSTH4fewD+EitXuNRnGF2yQjFAACXjWekUEjVqUuNww4hyl7P4t7485erWVufuBTfXofe/9m5r+rkcaOUmO9Q5L2q2XdGVEzwxuyfb8FqIsSQGpfs9ORF4LVZQbGGM7tklv3t4Exmp0v2NXXlKaxthGziQ8fKvDiQmE6RRP9VFAmlOUETDRbPpJb2UhHtPIV2LpQKqGmG9tAU7bVsKUvbMRXIP/EN/VbwnjvxT/wFvv6OZ589t07nb3fgr8LiTLZh+eYwKwYbcUbPpjiMI4KVxREL1f8PWmh3elpLfoI+S1c9oaXQ049pt2m3c8e4D6LLuUnRUDSNWxCdA2sEYI2dsIYZEbupUYY8LGApUEx1DKFbEambWPQCivUDpBfWooirltG9dP+y6MkKUWn4nG/XMCZ6gkvWaYDEQBjPdCQ/FstjeJXn65sUxaRXqAE0G425cCENYBEk4LuTH9bwBv9xwzp+9gjh57K/noszcMI67W16UpoHdlXIKimA7LGSQvlYnajW5CV2IQ9RDphX7C8+FDMpgB5BOexbR2/45BPtbdOrZWe8ZXDdjucf4MVYP4q07EeBkIMd7+NG3ScqZz6FzxLYQ3+2h15EMRXoRl2A2J/twVQHy9VK+sKSS6VghRTs3RXbjClW8fFB+AcEHfj0U9pf2/6JdKLsz+uxvsQd4RoY/xp7YwbLYC8sfQYt4wfQvGE0d9qBNCntDfjC59F29Pi4cVqKzid6fhU/lWXQSc2wGR40IywM7oXyUxoeK2XfuUPYSfeLB4hA2hC9AcELxIWdRZFxFnLyOAG0Qt9IUdgTvINbeeg+cY+o/YHx927AxG8LAyFq5ZMTemarJIUjAVw9xwoZLhbizBDA+PYBD+JSLNIUMPPGgm2mS7Ghp2cTAECvG09hDTcipOaGQiFI0zGtVzsatn/tb/2Z7SfnC0rqXlFNij8jKAl7d+799XcLs/IEV01iQpInT0l11aSkJoO5w59N5h6Bc8zqExJTUmM1n8SURnvPtLNBFTUNgEnEE8hhzTI+AJbnx1zJLEdszni9xNM5s3usQVYAJt+5iFXAwL36IZAWNp85KITP3E35r0499eDsFydxk6Ztr/nC7pwdZ+3x9uyqbRXTx89/s/1/1u2nGU/XPjht4ZzhVJKkqcNG7Xg5eqJ4QmHRTe1uK9+4dMjk6SOPLWOYZzXEAUlKAE1JJ6MN7GVHhvsA+EjI8BQ8YH01iWJczWAMd+uJgOyqV9wuNQHnwPTujOpG2OPSywh2JDkF3Z2LN0CrzDoNst4zyTF5jPowIiDJtLqyy8Zp+7/66o2KzYV2ue2a+1dXPb969rNZUkK0cvhd2jta1Peb9s2dQ9fRjJGTfzzg+5Dys0Yz3RsNuvMO051RRNeYeNDX+ECsSBkRkBYnYAQnS3edNqRFRz8eoMXjUhNBL+JCaqqM5V0GfRKxACIEWHEuHg7NqcYEjbslDEDMg4Ew7Pf6vCbIvbjRv34Zuf9ebvy2uVurNygVO8ZxlbPXH/0PZ849QTveU7ZOEqUFq878PXfvn0umS5L4aEkpLWDymAx0fGrI404dr+vhGeUhxOQhMHkI5pbyMARhsoGux6SR4EYSnKBvVhmU0ZBGnMko6rBCImYROc0L9LKepU/+8sCUDUUV46xdXr5335eVq6umrcpr9/T0qjX0vI/ytGjUEG7BmR9X3z6CBn478OPYEbRh5H1a9ENGxwig4yOQRzzQMYxEvEiCXTJISMWqm8UrxKpuGc1LPIlG+oO7T7QirLZ7/Swtk1WXjLKw2FGhZEMWhE0rBXz61rH+2YZ4/AHdnEZQ2+63jkeFfVXlVV3DPV+f/67223yOm7Hh0UW1NFr0Iw01fFKW+sofvbrd0rs/bU8nimmP7H4X9KkPEFEjdSB+ciuJxDOrwPgjWQAk4WykHFaJCGoDWCyhQIlnExo+rJWEmk0URuJ9TP8QkSVixJLQJVjYvsN6W6ixAacjtT41654M9A06E8JtSsZSTtMq+cMlVesiVstdkmlWeVVJQ1v+MNMTrT9fB/xNJXlkmlEFDIBmmGFzOpPbmpkb9GIVtT1jcBrsL83FsE9mKMZuNl1WoHYAbqcR3XL9co0g25ONyToTcDwZ0htA/2pbe/OKIFOeIr3a0HqnJ6ZIRw/eu7HIUfrDBwOVPum9H7256oWijeX7j1Y+DyqVm/PM9Kq1hkqVjthy7h8f/5odKM0I7Fi75JahtM2v++vH3UH/GFmpNXygx6YqCEtfgI14yAAD41jDuq9yoq9yNvkqb6N9cyE0cZvhp7CCYvMw1ACmTQy8GfNO4HmD+kyHSa6q7FJbuemVymUzZr6YA27ontET/vFNtJRbrTw7f3xUYrq+BTaVCfthc76x/BWVBAOl0KIB5dQbUM7GBhQsiQ2oLRUVFUK3c2+K5Rs34jXPP6L1p3lwTSdQ2ZUwsaI0BQvAFZdCMc5hT99VoMp2PTMG2ODSpeoOGfVRXpdJrCKUje2Te+2urr6hYyqefzStkAoV2shS0TqzUnjy3MTq7VZTeqxHtQZ4jHNljlhdFOtCIs6X8XYiYvA11Ud4OyvNMFZfuj4ktlofWlM5hy5/mNMG0a/5pVr/h6SEhpH0gKglRF8VOWf0P7CHJr6mkEbo0XppbUuFlHDmR/jOCsgH5oJdZGGuyHCLKwXrQGgWqCJKXBjtRPGB4Wazi2Xp2pHlYkUPVuJng6hY+lRzcDJE1w8lVQZ1UVLQgBVZVuN86IsCLSoyfqY+/guUyNtcoVaMt3XeUjmrOrPT9gVbdlU+MmfZCjed/tjsuU+lCd1q7hxbOXPq/O//E13KTX/7xa1LTElStIKbfuCl+ROj5pjuHwH6Wuh+I3VoAJfXeo9BjE2+SPf9F+n+OFtndbryauWyeXPWBIVufx8z8fPj0Ync8p0rF02K2pnu48xmAuznorkq+v83V8X8OEllXWNS1KIsAhjm8BEqaecOf6Gdrdz9cvWevRs37ubiAqdwsupU4BftQ9rpl13ncZoq8Bo6TaOes1obJYiwN4ylQ4kBa6T6ZuyCWApJQCwAybrtcC5WJGyOaWRO5xpgGrt0AabxGJxrxDSJtCWmKXV22cRAzdRNXdqtmrZ63fqq6c9ka6PELzYOK4lhmttvin7IbRtadmK/7wMq3DtC9/Gj+A+M/d9pZOm4/yYfnwKZg63gAgwA4kaY29K/IxW2RixglplbbwULFGGJs3UsMLm6S9zYiqINkxgWKH+2fbtn7m3EAnfcvuZsNpc/6FbEAj+V/pVzD52infsw5q+554EOF+RcTd5R76vHxYGKyI2tBsizcNrHjf4jjsTuWQAO+3TLMuUwxbzHWVA10Z/ncA2d8kS60K02bky5SSiX5k6O+mC9SYA9VsN6Hci8S9SL6GXrRaT1epHPD7gKC0YOI+80p8vuWjFODuI0mJIlKwmx+hFx+BpH0HUXHBtBb71+xMr1RZ0Bz5vUygVPz16377WPN78yvoyb/My8Bx6Y8tIbe7+sfbN8PKXtpPvGTb35xqmZuQ/NmbVp2O3zAd4PXTjlxv4lWXlPzVtcPXLoDInxPPv8T9wUcRDgl9tIxIM8iItBF1GHLqbm0CXWYYpvHC6Nt7SELtgMRHBAZMWpAxhZnwdrhruyC+Xs16f//POA3qlFme602/OmzgX4Qn3aTyXRq8YNFaWhdsfjz3FvwP5Wgow+F7rpfgwtUy+3SmZjk1iE8l5QhFLsrDDJ/BirQ8msKoklFSqx2kqzqlRRI6rNXlm5eNaStRmV46ydlcpN++hb3L3RZW9unjGe5869qd55N8aN9uBX98N+mtWl6JXrUu1n0dyglE2zZ2mlo4RuDZ/NncvnnXsTvno1IeIBuJ6PfGPMHjmcEIfwojXUhH2GVktT3sbS1L6bfj7dSmnqtxPvtihNWUS9NNXzvVND9XmEOEiD94qKHSead+7bd/IelsuaXDVmkwVy2cbSFfzZLJeFc5jLbufMFptew4J8treVM8HfjmaVLCO51YtYBjc8wI3Yq1FcCF4961A7Kfz93d93ljocnKUdLPulQOp44m6hWzTrjTe4L6NZb77JfXnuTe74669HU4ArIeB/LfCrZd2K/nd1qxCdqz3xCA3SrEe1J+ich7X3tPe4HM6jXUt3Rk9Gj9D3tTCsEQTMfIjJxJiVh2tjh9UeVmVEyfEFyHwgTW4uaJAz0yID4F5Fg4tou2yJXveglpv74HxfD4cjrjBu4MhAMSjAT/P5p88lTlppEcdw4uS/Lme2iDc3bGG61aKehU6IN/139axh3MPRJbwzOoXbM4SfeffQhoVGPauvNoFbKfUkaeRGAuZc63eQRCGPzQhBbLMU1JrZCTajk8wwKHYvIM3NYJT6gZ8ebPpTGY3b4lZFux4OWABjdo23gsQK+ya9rt/3/imrXkmae9/wO+4YXjEv9ZVVU7j0sQ/OPL7pVNGgdoceOz5pbVbOuonHHjuYe1PRyZePzVjK9hrRfqV+ViNLIS1bpa569mOUy8ByI6Xar9LuM33Y9yxA450xGtMKaolOo79AjQcaHQW1ziYa+TrFqvep3QaNfhIbbIjHqKc43KrVzWjsRRmJOkkoXpbH+1g+L5kscytH3nXXyPvmJu14rryionzVK9qu3IOPHStfmxlcO+X44++0G1R0atPxGYvHLp1x7OWTRbo8HqPVQj3vIYnkJoLo3GKtR73iUb+SGLHGXWnM3IHmZCyuJyKIZJNQFuylk0S2W1XywG8eQrTdmCbEEKjHE7+edLHk0fdY1cy/Pjn0qvHFAyaUrJ0+5IkhvSd2HXQP/eKBHTfcWByeV+Kcv+u6QV0Kp4/R9zjjvI3/TswmQTJDr5UoaWE1XqyPBJj7D2QY5RK8OcEJpwWWUQniRRWTDL1vns6yGoyWRgklSa5HKWAJJT0D6MEyl15CqbHaEpP1yFjY2d3yfqymKko8uyUrm5vxwd8rq97l+cYyynhO+MdTlbvf58y5R2hOwldfyu+tblZIWbrP/d1xP80BGvH+wo7sXqJn9fuI1FRIlxJDEQnTeAdfX0toimTPU9xhVn/1hmpsKZIZKAyy+1Nk7DwzdMATnLfgUyzoOxUfYoM2QHCbAoULs5QfFC0ePh3fhgVML346Ppl9Wkfe7no1E6ck0KoTEXmrksMAvWGeybTxjjScKQbJmnBmPtyLFuZc867tH5HXd/F8+dLK2U/Y6D7talM4n6cNg63XXmviFpTRtu/Vf7hV+ttSZY12uEwZv693aanz+0ol1kNaDvYWjxUCR7M6fa1LdhA7G4BzIYIM1Xp97ARAAy+vQwM/wiGkzc7GHSN2NppgtwFhUijiYJmfwwV/eUMMKtsdsVq/r0WtH0jx6bUNcGX4r8MyWk03LtOK6b3acPqiNrxCv8GQThWVaAfu06hctq1M20mvhV86jl8revgs437XHiTWNVeJnWEWvS/WOOeJVeYErNizRjqWzOGvxn5YGBnrW7uVtt0ielbDf1jhHn/+J/EP8QDEHj8g1FV6/FedDmPa0QcHmQwx4gGrvGWCidSG8yyZkAiH4WxemN3wWIAW0oXtIs5F8vTRxwT9Zj2lrUvN18dqO8Jf6SGlowtxbq3EPqkW4e19bWX3DovTx2emhPXx7TzZvV2Kc6eTjrrR6C1kvQnf7NiYMW7NksBLjKdVtC3NoVXaaO0L7bBWchudSAVK6WRtuaZpDdqTNGnHM09uELjhk8ZNmjVz8vgJwznhxSef2cEdod2pot2kHdQOaANphPbQ6rW5dD71Ux/E3PnatorNn1c9JU2ZVD2/cuGLE6ZJT1d9xmQ2k6zle/ObiASZIU65YqA2fs2kOfdoJ6j3HkfsgEv10JnaTG0WnWkcXHB/EWlx9xCoNSkDmf1qyCxEuuNM50VSqwWQgPPNeNdlJyahToD0lbah2sTu7I3ExvstL5BXCCQUDikhFxNLu/YA/FPBVwfbhkJKagux4S2YRSHIA1BsGXh7oTsV9D8HhNcJpwKDxUpYrgUREnxT6Y43GFxGjpfoo+fRRBq7naTMkOYakOYRXZqTIAPj6CQmzai2HKTLPVn1l759e5gtZVbhxqG7tg8aP+Le568kzehA/pY5M/relZY4rn/Xtn18Lt/NuV1uvUF7ju65+frb9L7xNGEXPSK+CRJor1tiLblEj0flMfByen6fTMN+ftqHT/Jn4PtWSWvAa5VoA+hKuKoTpz5MDP7H1SvOWIBnd6uY6motumgsLpU37s5m96dIRL8P2CTrFVU9ySoKG/OWJcNmDh6bekfcoNFVT2qrenYv7mCe29syaPDwiUw/F4B+DojpZxE6Kh/Dk/BrAfVqJ+6hOdqRTxqP1tKFdJG2yKMtajzQ50vZHKspnc2xui47ySoX6Gltq5OsvAf4c9E4axEyrPlMKyU68/SZmaGwLq56xclF+UqTi+6LJhcpbqjZ+GL0XX0vxhCj5DOkiLw8BC8FsBeBmEkWiYgYaSQG7ywFiljHCj7YDjaLLKE31MFGAecdwqveUWlc7sxPxoAcr88tmTqzulIG6dnq5FKgtcpSm9g90YKN3RN9heElRuelJ5joZNzgFeeYuC90dgjGvpONe7+DpKyVnWNJLCOspkL8CoRikMogIwVcS7oewdIZwKoN6n8Fm0hEXJWRjiTKCbYrkxiLepemcjbGwysSyeezgMnpsyMgbxmQRffWpkf8rU2PJBhZe8Tp9hUXtz5BwqTRcozkLRTARcMkYodG/eON/YA/gMwukZRcvCMcZ4kPqx5gOD4dIqn59tCX+3QW+9ica22i/ldi09YRo8djrcwpXWLjMR632PtnyNaLtz4/hjtYv1v8GvQbrI/8j37Xl+IP6zO6mdb6iKux490uzRXreHdi2w/A9gMXd7wDLtxtREjKwY435nq+kBq6oOOdkC8oSXtF1Y8db1+zjrfPVRPv8+uPpEhMSvBgB8vfrEoA51jH2xefmKR3vP0J8YmNHe+A0fFOtgFscaVltu+AsEXxymp+AWt+411C3mSj+W33tNL8zr5s55uFkWbtb6m+ttX29x9MaZp64NP3tNYA52+OKRGv9ytBFtivzCQjrtSxzGqtY5ltdCy3Y8cyI/i/7VkyIi/XuDzHqLtk95K+0sw3PwuBVhPfbumb6X/lm5/VfbOwm13uXB/sT5HYcxoSxKMX+uYWVf/L+2bjeRVXKPwzb9B69Z+2ZX75cj0AbkPMJ+v7PdDok8c223EqeohAGO9tUjJCzQj4v/HKlyYu5jFap68L88iXJe+s7kbw/jespYKMPSQB51YvUU1NvEQ1NSnml2WvHwzyv6qoMslcWFa9k6nlRcVV/iddDryxT5x594MkFly4Ux+KIhEyUDuO6TRtPCW28RovT/A24cYEr4mKmuQ4C7yVoL+VUFCbrOd92GdKwCKXLOm3J1yRtJhcLqBuIvPlFxEn9GZSiMX9UUzHAiSHXN8qYmnbmlW0M6xiByKWNsFsfYRYzcy64uQ18xTBInilwUtH91/qFvG/l/1KzU9w2uEpVw7zNiqCvCQq6E7EsB/JcjFtLSz+8rShxbdC26XtozltrdvISy3puqyxfN6Sphhm6A+YwU9ScSb/YhST1hqKSTesZTugmITEFKQnTlaTki8HaAwqWuKa61vs/mKUMLL5jpntCFbxNMHKYjr2dC5h5RmXsPKAse9asPKkNGPbDtz25c2huRguMIlvW1JwsW2ktGA6Jc8Lx7l3xTqIRHns2Scie76YLOjBCJJH0UvMYLTWWKlfv3eosCgMiXCO6fnvSr4vr94gHPcd/dbNxiTA920SltKz4iesDnAjwYK3XgxWfAW1vJFGJsQy/CQ9wzfSd3wmDoZudxz4BwuPrPBByg6JZVO11dfsKUh6dN5017V9S0b3u65kYGF2VjiclV0otu83Gk6MGHFdTudw27aFXZDWMuEUdx5ipAd3BdhMEtmwBi/G+vO1Hj2t9TAx1Vr1cgJrbeHUGc9G59i8EClWeZeRM+q7aioAI2gqmzD46vWF+X1umnTLDSu7FPQW6e33Tbq+yDtk2qRru1y+jvK/f+9FbqvwHST7PPCddRv4en2ItmnqFb7yotCL21qG87FLuK3i3it+fonY1fj8cCFEZfZco8Zn1MSeakTY4Dt7Ro2o3x7Dvu0J877hk6+7SghtpV21t7fq+7zMdS7zrJvhV1VMhi923FGjvW9c53wHKlH+v76Onz3+bnjnijGfUut7+zS8LwP2wpmNZ+z1YRZw0RP2dNoU0cUqKDbjLiCDTEWS2egGu+k0RnK4kfB5zYg3WKCvab/8msYt7bHH+RlrGqRgeUUqVqzslqiWz/ZDJm1vxiiDXTgT0oX+Qd3/V2vqrDTWDFeO2di5cswhmrN9m/YpfAde0Z/jPS93s+cJYSWmn1EREczhMD4KQBUtoVCzpwvFxZ4uZJSJ8UkHism4w87beBegAQXwZ9dSKi8l55euZ//pOjGBrKUNrIYUIFQxxVyYTZ8XN8cEJ+jCYrXPCReVPOE6pXCd31teR+FCxqWarkPxOkapqrSVyhTb002Asd4TD4KHhXwyBwnOMB6dptjCqszjhGItoTlWO8Na2PpIxmcpshP4GEUeM8YaR44VeyHtC5TcOpWTsP4JMvImABdTc7F+lIodjvhQJJc9zSWXWLAThLVRlGOHZg9pseNDWuzGQ1p+nfzGNL197WAPabFjr3rn6bq951j6aXPVxEFamKe4XDVOlwPST/izWfoJ5zD9hICGqactzulq1o/OYNVWfbQyiOOV5ILxSvavecbVk9700ksvUedXxZN7W7pM6br5bS4YPYo/724qLu9s6XJf96+0U5yvbGNZ1mkadDnHuTw/vpUDf3rePCHLY50u2uZ3jx6HRvHPCNew+3X8pFKvjELOh0+w1MMR3/iAL3zWjtnpgfScRSapzng+W+t38qArAA2o9evRy+/C2bpaZ1P0ciG6tdoNPBVgD+iB7M0D/+Aohw/yJnkUnbfiBtpx5CZp65C/SM+HX5TE8f36ae3pP7T2XKI2lFZHf6BzqTaPPka1qUyPEPh1Zc/UIJ3kgIzH597+f+LPPhMAAHjaY2BkYGAAYqY1CuLx/DZfGeQ5GEDgHDPraRj9v/efIdsr9gQgl4OBCSQKAP2qCgwAAAB42mNgZGDgSPq7Fkgy/O/9f4rtFQNQBAUsBACcywcFAHjaNZJNSFRRGIafc853Z2rTohZu+lGiAknINv1trKZFP0ZWmxorNf8ycVqMkDpQlJQLIxCCEjWzRCmScBEExmyCpEXRrqBlizLJKGpr771Ni4f3fOec7573e7l+kcwKwP0s8ZYxf4Qr9of9luNytECXLZJ19eT9VQb9IKtDC+usn8NugBP+ENXuK1OhivX2mJvqmRM50S4OiBlxV9SKZnHKzTLsntNhZdrr445tohAmqEsfpdeWKbffFKMK+qMaijYiRlX3MBRNU/SVfLQ2jkdrtb+DYmpJZzOiiYL9kp6nEGXk4Z3eeklVdJYpW6I8Xcku+8Ie+0SFzXPOfeNh2MI2KeEktSGP8wc5Y7W0WZ5ReWqU5mwD9f4B+6xb6zxj7j1P3eflW+E79+N1ukyzaV9kkz71+Beq19Dlp9msejgssDW1ir3S7WKjOO0fkXGvmJWujHq5HWdvWc0/pNxfUxWKTKRauBgm6YszTnXQ6mvI615TGOdaktNIksebePYEzZrMG88g326eeyVfMcMxSU6qk3uxt0uMy8OTUKA1PIN0g/Ioqe/W//BB7P4Hi9IeabvO5Ok/0Q0mU9cZcJ36T2IayfpmcUHU6a0K5uI+30inaIm/adUcsx802E74C0holcIAAAB42mNgYNCBwjCGPsYCxj9MM5iNmMOYW5g3sXCx+LAUsPSxrGM5xirE6sC6hM2ErYFdjL2NfR+HA8cWjjucPJwqnG6ccZzHuPq4DnHrcE/ivsTDx+PCs4PnAy8fbxDvBN5tfGx8TnxT+G7w2/AvEZAT8BPoEtgkaCWYIzhH8JTgNyEeIRuhOKEKoRnCQcLbRKRE6kTuieqJrhH9IiYnFie2QGyXuJZ4kfgBCQWJFok9knaSfZLXJP9JTZM6Ic0ibSTdIb1E+peMDxDuk3WQXSJ7Ra5OboHcOvks+Qny5+Q/KegplCjMU/ilmKO4RUlA6Zqyk3KO8hEVE5UOlW+qKarn1NTUOtQ2qf1Td8EBg9QT1PPU29TnqR9Sf6bBoeGkUaOxTeODxgdNEU0rIPymFaeVBQDd1FqqAAAAAQAAAKEARAAFAAAAAAACAAEAAgAWAAABAAFRAAAAAHjadVLLSsNQED1Jq9IaRYuULoMLV22aVhGJIBVfWIoLLRbETfqyxT4kjYh7P8OvcVV/QvwUT26mNSlKuJMzcydnzswEQAZfSEBLpgAc8YRYg0EvxDrSqApOwEZdcBI5vAleQh7vgpcZnwpeQQXfglMwNFPwKra0vGADO1pF8Bruta7gddS1D8EbMPSs4E2k9W3BGeT0Gc8UWf1U8Cds/Q7nGGMEHybacPl2iVqMPeEVHvp4QE/dXjA2pjdAh16ZPZZorxlr8vg8tXn2LNdhZjTDjOQ4wmLj4N+cW9byMKEfaDRZ0eKxVe092sO5kt0YRyHCEefuk81UPfpkdtlzB0O+PTwyNkZ3oVMr5sVvgikNccIqnuL1aV2lM6wZaPcZD7QHelqMjOh3WNXEM3Fb5QRaemqqx5y6y7zQi3+TZ2RxHmWqsFWXPr90UOTzoh6LPL9cFvM96i5SeZRzwkgNl+zhDFe4oS0I5997/W9PDXI1ObvZn1RSHA3ptMpeBypq0wb7drivfdoy8XyDP0JQfA542m3Ou0+TcRTG8e+hpTcol9JSoCqKIiqI71taCqJCtS3ekIsWARVoUmxrgDaFd2hiTEx0AXVkZ1Q3Edlw0cHEwcEBBv1XlNLfAAnP8slzknNyKGM//56R5Kisg5SJCRNmyrFgxYYdBxVU4qSKamqoxUUdbjzU46WBRprwcYzjnKCZk5yihdOcoZWztHGO81ygnQ4u0sklNHT8dBEgSDcheujlMn1c4SrX6GeAMNe5QYQoMQa5yS1uc4e7DHGPYUYYZYz7PCDOOA+ZYJIpHvGYJ0wzwywJMfOK16zxjlXeSzkrvOUvH/jBHD/5RYrfpMmQY5kCz3nBS7GIVWxiZ4c/7IpDKqRSnFIl1VIjteKSOnGLR+rFyyc2+MIW3/jMJt/5KA1s81UapYk34rOk5gu5tG41FjOapkVKhjVlxDmcNhZTibyxMJ8wlp3ZQy1+qBkHW3Hfv3dQqSv9yi5lQBlUditDyh5lrzJcUld3dd3xNJMy8nPJxFK6NPLHSgZj5qiRzxZLdO+P/+/adfZ42j3OKRLCQBAF0Bkm+0JWE0Ex6LkCksTEUKikiuIGWCwYcHABOEQHReE5BYcJHWjG9fst/n/w/gj8zGpwlk3H+aXtKks1M4jbGvIVHod2ApZaNwyELEGoBRiyvItipL4wEcaUYMnyyUy+ZWQbn9ab4CDsF8FFODeCh3CvBB/hnQgBwq8IISL4V40RofyBQ0TTUkwj7OhEtUMmyHSjGSOTuWY2rI32PdNJPiQZL3TSQq4+STRSagAAAAFR3VVMAAA=) format('woff'), + url('../font/Roboto-Light.woff') format('woff'); +} + +@font-face { + font-family: 'Roboto'; + font-style: normal; + font-weight: bold; + src: + local('Roboto Medium'), + url(data:application/x-font-woff;charset=utf-8;base64,d09GRgABAAAAAEbcABAAAAAAfQwAAQABAAAAAAAAAAAAAAAAAAAAAAAAAABHUE9TAAABbAAABOQAAAv2MtQEeUdTVUIAAAZQAAAAQQAAAFCyIrRQT1MvMgAABpQAAABXAAAAYLorAUBjbWFwAAAG7AAAAI8AAADEj/6wZGN2dCAAAAd8AAAAMAAAADAX3wLxZnBnbQAAB6wAAAE/AAABvC/mTqtnYXNwAAAI7AAAAAwAAAAMAAgAE2dseWYAAAj4AAA2eQAAYlxNsqlBaGVhZAAAP3QAAAA0AAAANve2KKdoaGVhAAA/qAAAAB8AAAAkDRcHFmhtdHgAAD/IAAACPAAAA3CPSUvWbG9jYQAAQgQAAAG6AAABusPVqwRtYXhwAABDwAAAACAAAAAgAwkC3m5hbWUAAEPgAAAAtAAAAU4XNjG1cG9zdAAARJQAAAF3AAACF7VLITZwcmVwAABGDAAAAM8AAAEuQJ9pDngBpJUDrCVbE0ZX9znX1ti2bdu2bU/w89nm1di2bdu2jXjqfWO7V1ajUru2Otk4QCD5qIRbqUqtRoT2aj+oDynwApjhwNN34fbsPKAPobrrDjggvbggAz21cOiHFyjoKeIpwkH3sHvRve4pxWVnojPdve7MdZY7e53zrq+bzL3r5nDzuTXcfm6iJ587Wa5U/lMuekp5hHv9Ge568okijyiFQ0F8CCSITGQhK9nITh7yUkDxQhSmKMUpQSlKU4bq1KExzWlBK9rwCZ/yGZ/zBV/yNd/wLd/xM7/yG7/zB3+SyFKWs4GNbGYLh/BSnBhKkI5SJCVR5iXs3j4iZGqZyX6nKNFUsq1UsSNUldVkDdnADtNIz8Z2mmZ2geZ2llbyE7X5VH4mP5dfyC/lCNUYKUfJ0XKMHCvHq8YEOVFOkpPlLNWeLefIuXKeXKg+FsnFcolcqr6Wy1XK36SxbpUOLWzxg/tsXJoSxlcWgw9FlVPcTlLCLlHKtpAovYruU/SyIptJlH6ay0K13Upva8e/rYNal2OcjWGB/Y2XYGIoR6SyjtOOaBQhXJEQRS4qEvag51P4ktuuUEzGyjgZLxNkAD4kI1AGk1Ets6lVSjaQjI1ys9wig6iicVaV1WQN2UiOlxPkRDlJTparpIfqRNGUGFpIH8IsgQiZWm6SW6VGpMxiMlbGyXiZID1ksBk0tasa+REcgrWbjua9k1ACbC+aMyG2RGONorqd1Ey3KvsMmr9WKUGrtEHZP2iV5miVZrPN5uFQXa21FgShu/bK9V7HCz4/+M4nBcnA9ltfW25z7ZKNs3G89bp3io+47JSdtbHvkX+Ct+dcfK7+Bdtpf+h+/o1trsvLQPQzsat2+pW5F3jvS5U0lhdi522PtbA9L6zn5efGkM/y3LsGAHbD/g22Tyv213N1GtoduwmSRzWG2go7BIS/cix/ameH20SbZFOJQFgyAFto4y3STgLhds2m2LIn+dtsB9i2JxWyA9hJ9fuNXeLF+uvtiB0DCWES6wxgl+WMN6zPWQDCnu6j/sUmGs+LuV1spo2wdRZrE4gkiiiLfNTvJRtgJ9RHpMZ/WqP4FIBQVAv5Qp3L2hFe3GM7/qa/5BWxg2/Iv/NsW7UG7Bzvdb0p326+Inb0PesfeLf56q+7BkDEK/LaAQBJXldHI9X96Q6+dVSX3m8mGhvy7ZdDbXSCE0YEqcn86BTP/eQUL0oxdIZTEp3iVKIyVahGTepRnwY0RCc6LWlF61ee4rHEEU8CiYxgJKMYzRjGMp4JTGQSk5nJLGYzh7nMYynLHp34m9CZz1YO4ZKfMOEQIRxSC4fMwiWL8JBVeMkmfMgtfMkj/Mgr/CkgvBQUARQVgRQTvhQXQZQQwZQUIZQSoZQWYVQS4VQWEVQRkVQTUdQU0WjmujcQMTQUETQWSWguktJSJKOVSEprkZyvhYdv+A4ffhZefuVP3WPRaUeiCGUEYwlnvIhkApOJYqaIZhbziGGpSMoyEcFykZRNwmGrcDgkfHDkP4WQhQ3EQBDE9pmZ+m/pK4ovGh2DLW8Y/0wRrZ3sTlWy/Ut6kPnlj7St3vzVJ3/zxZ878t9iVrSeNZdng1ty+3Z0tRvzw/zamDuNWXr9V2Q8vEZPedSbe/UNmH3D1uu4Sr5k7uHPvuMCT5oZE7a0fYJ4AWNgZGBg4GKQY9BhYHRx8wlh4GBgYQCC///BMow5memJQDEGCA8oxwKmOYBYCESDxa4xMDH4MDACoScANIcG1QAAAHgBY2BmWcj4hYGVgYF1FqsxAwOjPIRmvsiQxsTAwADEUPCAgel9AINCNJCpAOK75+enAyne/385kv5eZWDgSGLSVmBgnO/PyMDAYsW6gUEBCJkA3C8QGAB4AWNgYGACYmYgFgGSjGCahWEDkNZgUACyOBh4GeoYTjCcZPjPaMgYzHSM6RbTHQURBSkFOQUlBSsFF4UShTVKQv//A3XwAnUsAKo8BVQZBFUprCChIANUaYlQ+f/r/8f/DzEI/T/4f8L/gr///r7+++rBlgcbH2x4sPbB9Ad9D+IfaNw7DHQLkQAAN6c0ewAAKgDDAJIAmACHAGgAjACqAAAAFf5gABUEOgAVBbAAFQSNABADIQALBhgAFQAAAAB4AV2OBc4bMRCF7f4UlCoohmyFE1sRQ0WB3ZTbcDxlJlEPUOaGzvJWuBHmODlEaaFsGJ5PD0ydR7RnHM5X5PLv7/Eu40R3bt7Q4EoI+7EFfkvjkAKvSY0dJbrYKXYHJk9iJmZn781EVzy6fQ+7xcB7jfszagiwoXns2ZGRaFLqd3if6JTGro/ZDTAz8gBPAkDgg1Ljq8aeOi+wU+qZvsErK4WmRSkphY1Nz2BjpSSRxv5vjZ5//vh4qPZAYb+mEQkJQ4NmCoxmszDLS7yazVKzPP3ON//mLmf/F5p/F7BTtF3+qhd0XuVlyi/kZV56CsnSiKrzQ2N7EiVpxBSO2hpxhWOeSyinzD+J2dCsm2yX3XUj7NPIrNnRne1TSiHvwcUn9zD7XSMPkVRofnIFu2KcY8xKrdmxna1F+gexEIitAAABAAIACAAC//8AD3gBfFcFfBu5sx5pyWkuyW5iO0md15yzzboUqilQZmZmTCllZpcZjvnKTGs3x8x851duj5mZIcob2fGL3T/499uJZyWP5ht9+kYBCncDkB2SCQIoUAImdB5m0iJHkKa2GR5xRHRECzqy2aD5sCuOd4aHiEy19DKTFBWXEF1za7rXTXb8jB/ytfDCX/2+AsC4HcRUOkRuCCIkQUE0roChBGtdXAs6Fu4IqkljoU0ljDEVDBo1WZVzLpE2aCTlT3oD+xYNj90KQLwTc3ZALmyMxk7BcCmYcz0AzDmUnBLJNLmoum1y32Q6OqTQZP5CKQqKAl/UecXxy3CThM1kNWipf4OumRo2U1RTDZupqpkeNi2qmRs2bWFTUc2csGkPm0Q1s8MmVU0HT1oX9Azd64w8bsHNH5seedBm6PTEh72O9PqcSOU/E63PkT4f9DnaJ/xd+bt/9zqy+MPyD8ndrJLcfT8p20P2snH82cNeup9V0lJSBvghMLm2QDTke6AFTIsiTkKQSTHEeejkccTZeUkcYLYaFEg9nCTVvCHMrcptMCNuKI/j4tbFbbBZ/RCC8hguw/B6fH6v22a323SPoefJNqs9Ex2rrNh0r2H4/W6r3d3SJ7hnrz1//tVTe08889OcCZWVM7adf/Pcg3vOfi7Sb7ZNnb2MrBg8p7Dba2cOX7Jee6fhjy+tvHnmqCFVJb1ePn3qzYznns1497K0c1kVAEgwqfZraYv0AqSAA5qCHypgEZilRWZ5UT2PYsgNdAxLlEcNYjwKajQGgw8Es+JcAwHH5qETLIgby1WDHhpXgAyPz93SbkOsep7hjeL0eqNVIP9lTHKRzEmHdu0+dGjn7sPHunfq0LV7h47daMbhnXWvenbo0ql7x47dmLCSvrRSvDNw6uSa3oETJwLthg9r37v9iBHt/3lj9amTgT5rTpwMtBsxtGOfdiNGtPujmzivGwjQpvZr8WesjxPZUAYhMK1F/0qJXHRyLXWOAx0H50dxboQfxapphKtHGVUGHf1gc6PC6GkIo0NCsYGDIdUo5n9yHFb8Uz0qpyqHT8qpyOmZI4w2c1RTC1d7tc4anqdBGhkdmshNVo7GA2MF8+opFMrXcvAt55yfJNbVj8SKVhCJpBCfz+vGL5mK0yVjQRtLLX1+osicbALyzY/jkdK22by5e7c3z+x5acqYSaSkScEL3Xs8T9l3/Qc8NvUqY+SjNsv87OFG3YpXpZYUzytzDe7coy/ZsiQ4Yuzd/U688NSmCXd17sZub3v7oC2fjfhCGltW8VnjxjpZZy+dWjwpIJwormzTK79/iW/wBAAgqGEiyZKzQISGiQpWr1h4SISYUkm57FNqBQIBVkr3y8NAQ+3D36A4IWQV/JmZqJw2NT1T0Q3QAqTsQblg41NPbiqQH2Iv035kK206mGysZG3YMSs7xtrMDAyhTcjWSC4axqy4LiZRQdFdvnTNq1KX320HjVawZx6SCzc8/UKgUH6QtKPt2PKac4MDleRlMsxKBpFXpq4ZVBNmKyIxHbSvMAF1NBWyAQPW6z3nEIpfMhe2fL8kuIX8TClDEQQX6cwueUmTlNNpRPey/31uR/D0LuH14ccWkqFs//wTw9hv00gu+7IyEr8T3Cw2Ex+EZHAAktOEiPrIJO5s8hWcNqema06vU3PT02QFW/8NW0tWfSM432N9SfA9chuP5WOfkxnwHUgggyki+HwUXGw8M+65u8v3uexl0v7FyJpdaRIdRN8AAdJ5nYKQIGi4CB1U8zNNoUnPR3X1LjTb4EsQYnsMWACwJO6xk7e4bT/99GX0N7R2ndAo0jMzAOfHN02cnKkT94fv09bvr5QLAD8UpuJ51ev0rCK6SgOc3gCn19OKL9lADWokUbkS0ldBzwNNU8HdEjRXVGu0qPKIei288y5jBN59h9Cfl8yfv3jp/PmLaAn7hF0izUgO6U0cpAW7wD7NP3vy5Fk2o/rUyQeieM4C0DcRjwS+aHYSJiRhdokFkVRTjNUkvr1gffj25dM3f2ZXqEN85awnGncAgOhB3A1hQDSuhqG06+MGs+MEg0I21x4BImqiqcGk+kF0sY1xoc8M45pOL4mpgk13GVCnJSTTKXr+KSPXFgybNz6w4msqEctn537ZcSt7XKC7j1Bp9YE+E9bvXiU/S5K+eGzlJwfYcRkI9MM9smOuzWDV/+9pGmaYlnq9hLYFMjf0Fje13Izl5ntACdyDxkxTg0pcymnYlcImJDTWkK0ZcHQO3nrRBvWETcbdrEfVuA6VHa2IuhjrtnyGTjYeWzR1zsyJK7+iMpFevcjmTVuxkH176VX2rUy/Wls1d+3ilceELgtnTJs/d5R85OMrL40+Xdyiev7Ln15+Uh6/ZNmc5Qsj/CwFEIfj/jeANOgFJknoJonXwOrVZBeho02iBmkcTDlsEq4XIUsyjQo+3p84FpvOj7aLuIlTcynCvocf/qlml0xn/1WziWySrVR5nj1BOt4mXPlnKO1Lm0d5sxb3wsB8cmFylDcEVyexVFLRSeV8JAmXnJAllfClLUX8xpYRRhu0x6VoUYM5CS4WP7Qol4xGbc5ACRJ8Pr8v3WalWOW2FIsc2wbl3kECqXmlRfO5Xd/44pfPn2a/S/TjFRPnLl42d9J4O90m5J9jt9zYlFL2x6eX2A/nn5Us0xftWbf+UPvWQGEBYukSOQMu6B+nMDE0VnSsHA0kECeUCrz7ItigIy5ra0J7xQK3tGcqRoQsNh92U8w/JhEZmLktBoMe7bO7rLB0epebg632jH3uY/bP+ffYx6T9mVGBvNsWTF8WkF5wOh7Pcnz4lOJvxb4//z77iJSSLGJH3RhW06N96dRHXn5ww7qD0f3pDCC6cX9ugKIoomQEkXw9VczkxNMLnBCUCoruT0/3oxKL7r/NJmk/p7m+evWfGuE78Vt2lRns9N13kx40+4fnAD8CjMf6NcP6ZYKOq42NrmfDJWy4Xj1P+cEsSLLxkhUklCwkOAq4oqQVOOpuIs64nGxq0JVQz7ij5o27pAixmy+WM/67KC2ZsngH++XyNfbLtqVTF/36ykt/vrFletWG9bNnbDTmjRwzc/aYUbPF4lnHCwofXvLa5cuvLXm4qMWx2c+eP//PkRkbN1TNWrWa/j1u+eJJExcvjpzFAYg3s44vfRL+t0nkS3xjCynWFA5OSSRLynVkyecXVH67ol5PpINovJ8YLr/dnoHXLW8MFxXW7i3ZMSj8I0l96SOSyi5/3XNvxxtbB5aMDNy4dsmE9UtPPfNIx46difLpNfI/7DL7kp1g37C3GjV6NCeL/NStbO2ps2c2bD4CALW10f4qDgYDNPymcCtU8R4uYw/H8WnY1+/HcReOEKGKyJDmBj5OcRwItIUhwnqhFpJw9xFg6CkFlTYXTfVqZdf/tfIcAE0d79/dG2EECYYQQBQCAgoialiVLVpbFypuAUXFWRzUvVBcrQv3nv11zxCpv9pqh6DW0Up3ta4uW6uWCra1So7/3b3wfBfR//rVcsl7+ZL73nffffs7HTFBR5D3WpvCDmUdIQb1I01myQTjoQl2MRpRl/r3hG4oVpCF83Vw+kdwei2j93o4WagRrjD/Nw7YgU6IrsgAfQGRcYCTLxUZur5kPuL/lYuuNgU1XoSa+ueEfPon+J1yrD1J7UCC+5VG3BHBHVHcEcUdlSGKO3nPyzABMdyNFOv48MTEyEXCyPp9KK85NAqGGrz6I7y65gckiwz3dgAI+xivtAIDOA3LqyxbS9V3By2ZYgWxj1KxdrMPUEhIZKJWxzrtdWqXG6lJNABmTO6TO6EgZ/pvgvDn0c+vb5z6WEvxzh24q2xeXq9VAwomDR8q2098/X7JuWGdhg3GY64xvHvgZPkLaR2wgixCI1vHWKJpbdGx3G7mDCO77O7d6Eeg+9T6IJEoXP9qW0dDeSvNbVsrcjvaUN5aC9pa0c2ZWrhMKvyhjOgmkGUyEsFkpRLVKsh0dyc2B5YQICBgIe/NBCIEGNktqHxMBISRCV+50v3qzz2L/GNX5i4ra+5/7cXJK/oKktUtLnpWmZsBf4zfwZ/i9d7NYU+YMLgiIyLr7Gi8AA/zaQ6/hPNgCdx2D3ukdEseEwlhjDkuaOZ8eO9b/PGA3n2za6oggAlxCaLjSGGvi6/CKXAHfhxvwhtxbhtLaVQsrIM2+DLywL6O+mUrO6a7GfRIcPf8hNHZAIBE7VQd8ASDAWfec3ESdiGTC5nSGsiiwiLUtMnjuEOk1kzFcI9JHoR5kz0Y+SwCsXdhGH0VKhzHp/+FzFeRz9+O7fCtL2Q4AL8u2e72RcFosiLP9wIgHmY+hxmEgGJg84/lVDxnGtpH+FMziw5T/GGx/Sx9V+NPbS1/uvSGcm/t5vGnTEK3rUG9y6yEYO1+tfpYOon3TSpILhmHhztfw/bCn2qhobiwdDW+fQN/CjstfKZ4Dj4A9dOWrFx2S7KdOD56V0TLD0s++Qptwe2eLpq+6O1Jo56aACCYSGT3GbIfW4Kuj9KLgIabbN50LDdy1C0P5CSL2U+190OAThfGG/zHkIjP1Tfgj2ByPUSwrYiu7925+a0D27bugj/KF/F1OBh6QhP0gEPxrZ/ljc/fsONrFTee28R4g67DL2Qd3IERJIOHLwGln4cGSUJdTxdyhgDi1AKL4NMYAdkLvyXzDscv4Os/X3r77Nm3JRt+Ef9xEdfgl8Wb97668d7lQzcAZDjMIDh4glxAaHWfDV1JZj/rSS1tOuz1hHmUcIAjHG+MklgeL6F9LCbnn+jtWIJ+rI8SzjpaowWoDFuPSrZKXAiAE5+ZjCY9wHwiifwfvmXsI9wJMhnuBBn3B5CRXWYPc85tcJTWCd84gtBCVOTYSOfNYvNOJnxzgfBNCMgDJG7zSAeR2NXUTWzOuYmcC5VObFq7NxloMKYVZwDIYliIk59EGoTQ8FMi1WHihc7472r8D34dZmIIYUsBXXXbuXHroZP7iteG4MvI91jOCtgbusEO5K+347Q8e+MPb+JPbT/Gt4ZtDjppKBnYmi4D3IJyT8WxGL/UbqKsmPH2vW7kQdLd4LSKMre9bogIAvLe7u0GiyvOul0mNypGuE2h989SwFg6lJAPH3RNyQJYyWiVDLWO6XV1aHWtQn/HIrSI4vwGGfYxf74lFwHn0WS/ZYX76uoIKFu35IbrwlVyYQCxLpa96kTTx3OvJq5zuRfv5Pnw7hyqq8P1Z75rABK6Pm/yyAWS7d6fZ34//7k8f/ry4ka6xjKbeygnyTXR9CbFOhNBTIUiJtZlQleZiHWo4RgPKCvqPoxRivhqEFpQ55fr6lbBkzDE8TtKxt+gmY6VhGRb0QTHkw6dul8oThJo+wjtwodgwulWsMINaHf91LqjZPMpvyPTOJQPmKOhI8f8PFG13EQvVGfduUdgdUUc7AqJkgqDxNrKgaMhs+eobTNFT+700efrUV5FO30KebG5Uc8EWtlONUbCMKgzknfwPPyXDJ+HyXX+Mu77L9xf9q8jy7JPHHm3L/wDzYL3tomF0LEaU3YHPO9P/D/xPpFcNlR9sDfKQ0VIyDvYAkWjZCRQzAmOFb5urd0QeRq30fSlk1sX8kKZEurossFEhcHnyoTDl8u1YiS69x3B9zwSWwMExpGYerP/TAzKwmQIe+FjUFIzXI7/xHfxIdgdStAT9q2tfHHfu+/uf+kjNJB8sB+OIDdl6AFH4n34L3Twt98O4jvvXP/tEFB10nkWhzCCLoBffFVBMRMFCoqJUu7Jo9qcQ5WQhel6UVXuFrihDj12C/rgmlv4Xfj4imeeWYHfRW0c30q2f05/8nfluilTqH6k9PKT+hJ6GYEFpCu4GMj0BlevUyth7YJ7K4qXwVBu5hBhkW1IDMiHUy53QO1z+HbC7IyHkG/FrwOur4fAz/Q/oGEDoWEgCAODHkFDdtGcXDTnCMq5zh4tAL0r8H4kpavGhqLpIBNRJVTz83QOvA09Zkyd91RIxN025kVT8WEYuGH50hX4HMp1PC/ZLpyZ9q+OkeWL52TMDTFb1nadMXVp5dSnJy9Q9tJwohNfko6pURM+HNWSXLSkiJtbsnyG2TXfxfFwS0N5+AN5LeLfk+CaalbRx3ANsgkVK167jf+BYVf/gGESurZtzbKynQeu38YXb/6EX5bQb+9sXLEFzhw+vX3GF6/ZfsL4bXnqqum5OZM7pl96/eA3tz6Xly0pAhAEAyCWMjs8lpcL/M4jdosEtVlJxXhgirkUP1GHnxBHE/PJKN6sVGi0nNDoFpObCZzc5HQCL2Jc1JAPCxfF+1idfOgj3sJVDXfxqbrX12+xS7b6DrXYAcVbQnV9h+07dmwXqum83gBIErOT0h6ti1Svgj5NhjuVyQPgGCjm2X0hcx7M1kRooc4DKgqUA2AuFBx3fnH8AwW4oHC0GH+3L9MPbQCQf2TPuZTjaH4+bo9y+oEPGxL9IFfbfYkSzHAPk61ylpwjE4wKyA1qmgtMS6QQLWHPpkMRHYZTpdFCH61HFGtTIrRCc6KRuj30nxUBCMOOwggIr9bgFy/iizK+cAm/VAOXIklse+9LnYfY9m5f0XTvOnueTgCIvzM9MZCzvDVYu64bu9CRCx3brjqoeDokgUJH8jwTKfoEd3emyyzq/2glwTUEZ8DP8AVcRf5dgafIVSthCwp0tHeEojDHRXQJfU7X1YvgdY3g5QZ6cnhpZn/AMhdEigqdGRClC7oCqqHAaIAYNrITG6pOLWguHAm9sa4We0NvdANV1WdjiPTC83TuIWTuaYynHgfcdA+1JewiQCzqxW0bu7vEwj/M0IinwRkTnIPu3PsFfeeIFu4ePbpNHFi5Qdk/S/FhFCSvBTrQmuaUyJS8Jc8JFaXYgdrxKOiFF/B4uE2q/ueVI7rPld8ykZxQQWNOCMVqtyP5KmUV0w008gZRM18weD0Rhy865yaANFUl8m6WjsuY0hgTKbXQ00qBl16S195pf0QeDCCIR+eEeMWP421XpZaC+eZCZJgOCp/C6Ndg1Ccv6GU9Ooe+cbSFuxMSGC5CQ6awjXnnQZr99YDpJtEo17b6ScLmDz5g3+srHkZm6TgQWX5HiRfY3yJDRTCIBYg47TQ3EguI536ZvstWkibUTqdDOh28yXA/rXTQWwwWY0Uhj6GeaEHmKuxAUC8ehqKsxkeh2AeEgGiwWcE2gGAboOcEjmscwUumaSUSSa34wOusF7ELa7zgtAz3Eq8yr71eb3mJxRXZXiO8iEdB7xAOrvFq8ELFtgBOj9h9A2RmQvMxZC8X7WKJUKJJLHRs5YNnVN+bw2mwVVE5gqeXj9DpX4WvvH3n+yNj8nJG/QZ1dZVHfm3u67iSu9H/o4mz+7XtE9lr3Jvbdr81YuDIvunyouMfVuDgrHnJb+Ym75vQPe1JgMAiQpME2R/4gGAwUKMtfbWiT8+rG16i0GSJiTelgngLhgXJdNQ9YHkGH0Vr6nz8lGBEwsWThZs7+Z+p67Q67/TFuukL+xWFBE/OWVgM/7mJL/fPXi37O17q1oPIn/pXqp/IwJ0zu5dvpTzUj/hQf4p91JiJYsfrtbKdZ0SWuhGqaWbNl47lZtcYt9XsR7Q4IgYJjeapCp5GttOHzr2AJNzwdk1DQ01lnYguzsh/trj4jQnZ8rYLMO5G2HUY/+Nb8tD5J7aEbT9G+S2H0FbgacuI5qslp57XMbyF+N/R1mhgQUdaSBWpROetTo9c8c9zLp0csspad8Y/bkPBiUt1Ty/oPSk09Kke82eiZlCAqd27oJx/fl3eKxuG3thi75IKv03J+uxltleGEtreEbOBH8E9T4O73nV7BAEdZeygWHtZEPGuS4LKSMkHZ1u7BNV0LmSXQgEhNzCTBJTJoqM8wQKmAuEQs4Xmn/pexTXQ+8x31xx5SF41b9TqzD6pp/YPm94MwTcmmGDMjTY3YCLEf18ukxY/3yFmb0IPYV/ZZClgXCmAIAoAdF6OAWYwABCWeJDuRnJhdH0qSmjIJwC9ubggrebyI0KSVbDRzapJptHE5dkXXqi0hT0RE+DbMSg7+8IFYXnFwgNHPT0Oi/KwAQsr6udSGg/APUU3xr/RYAxwRc2F4HpyofdwXgSSi0CKp54PAwby4oU8RZsm2CVRiSCw7A2LuzXFOgN+OFmw0ep/CuOb2f/uEZeyvvfSudZVw078UDdrQZ9JltBJPRfMIVyEYFpOnzX3jn/2U0z4B8Fh02ZMycwi3LT5QGYqPJ+c9flLAAJilot6sg+MVD+rvgO/CzihojXInKuh50RKgiIQw3zY9lR82KkJO/Nf/6hu7Nju08Lr6oQ3ew0494OjCG1eVJwcV/8rmZ7x9ToA4BJywXI2Gq2nd/VxkMEmqbVesraew1m2uISWLYqdoftXAKAGG+4J15Lf9SZPmcFJI43RQ5aP2xlEDvmoczRX56C2taxZHx+WMFn77outO4c08+lkSut+k858b8WBSjf3o5Ju4DBxDkMDQLAYADGF4KGn/K5OzFVO6h8d63FDSqznvw/zwCtFtbWF0Ae2wjuJbXEVnsORsn/9UriHpBTszLZR6c3Hx3ybjo8RkrJ1YvkvIM8geyMcjNY8h15r53Kblhej/DZRLsLIRRgz4vk9E0xtHTPjKLMLX/nyPAbzveL3TZi4LaLT85P/daRuxIg+T/mjuoL8HuNakeVY03vAyJHDxl7+0TEdrVk5dUB3bz8PRxZas2zGY3H1V8XOynMtBED0FPvQvcA9F/covAK7n5yjFyIXDlRR5xHNbRa/v/CVI3WF47pPbU1w25WT98k5xxD04txx6Yn1NQwZRT/FEVx8QBhIcsFGTR5TDerHW7bBfD1eIpnfTJ15HWHaSFrPaCZsm0jj+ZEEIx1RQ0uX/3xt6bJlS3/5ddnSurTUJSXpGRnpi0vS01DkrZ07d+6oNd3eQXzEuj1jRo8es8e0c0xhYeEOhuMiPJLiqNWhbIk5TuCkhwdvrPxP7RPK1+Ym7ZO4S8dz11rrPvGP21jw8eXaBfN7TQwJmdhn/jz4zw18qUuGo046/0yvvrgSO178IrMzNj+W+u/NjL54pFDvxL3/o+S7qvI9XLj4kYir0pyg/hDln7/OGnSsrtMzg5ny7zEuNHR890bl3+fJJXcjkJyaRpX/weQkeCch9auXnXsPvUPw9gbdAC82VEWkd42p6g022CjAKkbAKTSA6g71itCIdMpo5y5DO8d3HxFYd8nQdvEAvwiDMEJMSXQYxM67c/J1EoDUThfOkvkjQZnGItW7xm8EFr+pGCpMEIjZPVNYTl6U6qGKF5sdbEbu6ZsFkRf7oGbEWTA1g9NYcIenqJmL9dhCq+1DQ4kTIoQaQ1Fe09EfZ12Ha/SHJYETrYxp0JWRS46euHr4+DUS+hk7dEju4GVnjt069sVtGf0gLsrNHwsjknoEtd1a+syHlevkrJHZjz2WFRi1femGg9+ulvMHPaHICnPDdbRAygRm0E/jU1M6qIUsetcINl/YRG1cN+6BaXWTL5V4PtRMUfjFrLgcVKv5wDePHu3cwTfCJzB4UPvl2154QcrE/1Q4Xs16TCfbfYy7X0aDKqBOwW8ekR8eYmcmy3iGVrU37zloTa6m9Hq4ExGrEzGqaYVQ666xb1bV5uYNmRVa9+WeQXmXfkMrHLPWFqenCM3uHQcQhAAg/EnwcAddeCnGMS/v4iESE0etEalOtqIslINICfNI5IwrKdEZK7zTXDZ+cw8v+gIvvAcnDxmCztw73ijHwwGQqsmFASzmrAiNNqUXTdsBD5j5Is07sMBWhiedOQvSvINEyw6IL27vRWtW8nRFOsLTQbp2OppBJ7ds0FkqxxAWInU0nW40G61ikvzKNfztiasI/nQCf3vtDfn7cpgEBXjvOPrRw8PRUuzs8IDobwCBBQDhJnkOT1DM8RgnXR8VT3LXeTir9kC1PZy65WPp4EuHAWSgnwjVdCSRpmgZ5h3sIQ+TJ8rMTzdSM0IQ6IjEj6EZvw7z8Y3PPsO/wXzy3hedgE87rjku0speFIbMCu0NuKdQT3A2gWGcVNVUOel5VtNwAhWxRkrug0pIkSz8KEjQdON5kfIBwU7W2GGJNN74i798E3rgjOhdZa26hbTw6qDvkh3QBs+C7tD+FLp9L3TaPr0biTgMSx4lxgBIdBYQqihv8nvkPxKbKiWFSetRqOOa0OPo0b3om6odCn2S8Da0Xk4FrUBbQMtjQCxNiWa70doHMnC1gmadmyKjnVH4eJaHZzLBpInSo4LKF0aMGjXihcoOo/oNGjx4UL9ReFviH6+dHj/dPn3i6ddqEldbXp5/evz+mNj9Y0/Pf9lC8XgT18KBD611htTiG/jSS7hWfl/BuwXBe4YG71axNj+Ctx/FmwxaWW3Xmf0Y3uYEBV+GPlspiq/VFKqg36IgZ2he3tCcgg5HX8wfMyb/xaPfUTwn7GsXvX8SxXN1Ys1rpyeShxh/+rU/EhU8ZsAl4gUhFgSARGAzECSaqly2GfjqJxb7JTdtAXRHKva7oocjFffQaU1csC0bvD4ncUj7lAGvvr5i0Na+CYNikweh37d+mdm9fbtxT/ht+SSra4eooh6Kv1KGV8JSsTPzV6IYFVUxpqc6EFC7nBb1y5oKa01zVSn1UvBKoQrC60puxFNokCJAGJio8cU4ueUaM/GkG5iObmz0uO+xEG2ivTBV0zGQjuUtm4isKF0/LLjCuoL4+MqTQ+deQsIH6z/+6PTpjz7ecVBAlxoDLNLiMy2v/xoMIz8Pq4ZtQq583/KbLVJjoAUS7QjEiSTfEwoKwH0R4JpG0O4m8ih2i8SqZC2x2gwVLZGw0AIbe4CvhX7s62otmglX0S1oJYwXSSgcyRsDZrIvf5FiotBX9REesbHSczvdf608+5OIrhcNHDTKHS5DQ4r7b+t89KhXef7cyt/P3jxnlycULpn5e6Wy3nkNP0vZ4i1WsdoeECXPB1Uj+QLUmAe1Z6QuUik9TYxMdNpbiWa6jZVEoi+xGZvHxxGTF4mpvQ+NKXyn5+I1Kzpak+LXrVnbw1Yw0t5z/dpN1iRr7Kq19bNrXnu1pubV12ompXbJTF267tleB0YVHsreuG59Ykpq0qb1W/v8e0xBec8169G8QxhDdOgdCBqUPRQIgPg+2ft+YKqyJn7kEfy4TGIzrUFJVYm3UYi2Az3d2OQ9DfWSwWZk7Gfk61bkaqYa6VjeTHPfw5k0sJiUf6SlTvkHLegpmAW98dPQF++Go/HuOrwTFpK/YDwNGoQOaJEjofLpyps3yYBOsbV4hsivIqW/ka4F4KuM7FDZezDWLsmAvpNiK7ylYAnRsnCy/ajF+8zPP/+Ma4UW9T8LH6O/AAK5uLW4mvCqldjWs1hni+qb0t80u4c5c5Kp2tywOVWtjHexYe0dwpSuLK5Nyt4ysQO9G0Z788hYHt1kpTJXru5s1yMjTW6KvHkbzgLTyntzAgUXVw/tn9UV1/zyA/6UGLmvzp27evl7tT8P7p/VBRqv/g71JMe5ekHp0rlVt392fBLVJzwxfv7R+MdDElOegSfyVkZ1Wlnw1vFT52U4d/Lo3r2HJWW8++aw1e06rSp45dPLJ+XC5YW9Bw2K63KonUdAM9PAzkOHJxpMnn4DH+tboOyT58WfhDnOtWnFMjCwmppROrVc1VtHDH5E+YHsUon8CXNqa3HQrVviT2fOnKEZi8GkruEHqQq0JPomHsxQ+DSGLEVMI2tayYWV7juLeJ/HYkjht6hR15ZISmox1u4ZaVFaRu0GT5G8KzeKfIWeqFkgkXaTskI9ZvO6+BTO6vtwpV2H9e4ISvKfjeIgJNp27ztyZN/uchFtGjYsv7Awf9hQhzcc/OdtOBi/cvsv/OpcuAe2gZFwDy7A5/G3eBQaIG/d/eVbs974eu9mOX/gymmzn342Z+QyfAdvhROgG9TBcXg7yVknQxvui4/hKtwH2mkfAqoQfFiNWTR4i1Zf30+dUJ4tkWnqhg4hZKCKCFSz9IemXlYvs4phfaz9sp4UZQXrY/WouCJdn61HJJdyRn9Bf0NfrxfzKjz1LfSImI/6gMZ0iforzMmMaFzfDPcPI6ojrkT8EUG+BSIMEWjaQeVamHaQXodECMWEvk1lVCKbzqigkW4egmVKn1mlrzz3bPJjXZ54Acqvrl6+W98Mr7BOav5Mj5zO6KgpNjA2de7EKbOtaZlxsV7yqNK1y/Fx65Co0s5hEzLaR8coteujwAxhlrAJRIDqvy4BHaiGXRsuAQhK4EzhqBAOJNCccm25IPBZQponO/qxY5mQBWdC8TX2W86+NCTTqlwgqnzrCcygE0gGa/jMNl9j4i1y/q5Jw4MB3ibW8BtbUR1wJYDk3FqYvFlzEVmlFiTdZg1oQS+tseX+mm+F+luVNmFbdDWpvKZNSJ1FbVhCw6dGDf8qpR9+TZV+RDZ2JQ12Zdm5WoaGh7fCgK1vpianJeo8drqLWb32lHXN71NQis7xPAtTXHj6DfyW0H9ZSfKw4KCneia1zTQZTP2iErp3XZ6a+ERnpq9WSM2FfCZPDLSLievSpGuS72iLvpGa76Gyp0SwoVXSMUb/ni60d1flz1l3wugfuJ91RySF6U52ByBD08vBtwwrkQRNF1HJzqJJ27dPKtq56sk4a/fu1rgnxXcm7907efKOHZPjuz+ekNCjB5OJIxquCXWSB8HLG3SluoWL4hHF0WQXpV3ycle0l82LU6Z8eyUkI9pFl+IbvAOO/QaG1x8RsoSVJ/AMuOoEXHT3chWl41NoJ/pKOgECwRjXrgKVMm8B2ssAYLGS1Z1C34XQevFAzV5H1do2A/SQTj6CFWyqy4CkjtBXjv2wY0Yba0JqxttIfn39qp0FsxcjmI92rocg4fG27ZJSOsjj1pfO6DdzwmQZQDAKlaHrJCcdBT7URBoJ7uUy0liItFCCjoHqA10OJE/wViD1UwLJAwXTyyl0KKNDOh1q6AfZdGhQgOkzk2+Uh2qkZFQosyiiyP6LgsUHY6PSo7KjBPKVKMJK3lHBUURmXo6qiSIC8gNyq7ytZlv6to2i3w00KAHtTk0QRY1SaRsB4+H+zNTMtPh0SqPSza93T328Z8XmFYdk9Ha31Ixe3bvNE5+O7xAZ3y5UHjV71uTE4QH+I7pOnT9nqhxtjYtJSlyi2HuzST7/cWc+n+rCdJHab3RooEO2SLP5IqULeVdBE/VE3rxFPxpBB286XCYf2cD9fD6gpQACaxQw05Q+9EK45oh0XMb1bM4NJDYczOIAOeAh4XMuDuDhEizjC328XZtzNEEopkJYjBguHVMweErLusu6mFk9U0dH1JJQyqaXZqemCM3vHR8Un9AiCKdJ5xWapAEgTGU1ia01cdQHGhUQUFxwstVCAW2vsvigBTnXsAMK1+DjyA0Kn52F0t2+7Df3of5wg9BFkVNC7H1yKXYO3FBbi/r/ocxfhDPhSQLpDTowf9pNZdipLAwgcnHCZqLWl3AyS6RiGibCNM+MQa/u1qX17NY/REjw7N937Jxn28W0ay2tUuYajLbDLUQmSqAH3wf8P9j3XHewTeC82LD4cLjlwxKYjrajki1mJudmEXuknbMeNQOQFeREsL3Eg9ojdAghA033uB7p8D89p2HW4T17jhzevffIW0MG9h8yNGfAYHHmpvfe2zR986FDmweOGzdwes748TlMR08EW4VVAjE8wGd+AOjAZ3Aqu28DQLpMdHUkOA+Gom3k9XPoD4heAt+gdwEABo5aBB/lOzKQqhhsOHBr/C75zjkhmn6Hr2pk3ykm39klnWDfOcu+840wi3XNfQsMaCf9juposO8ABEbimcIXYmfWA9YDEEl9v/NL///p/JJZl5eye6xO+zaOdYPRQ03Q6yh9ct9h40f3m45+E+CfH35xfcO0pGDS+oV2r5ubm/1sTsGkXNb6dZi0fnUcPhjuvsZsKqUnSReKIkBr9mRZ0APmAndwwEsSxWjySCqMRYWZCT+CwymMwRWmuwpTBV6BQylMM1niYUarMMfB6/ApCuMtu/yOlwozESyHecCbzEVhaCzIi4hiLe5lKuwxmAEPUFiTRGFNylEwzLdp+AsA3WDJxnLJW7iqz0c1PwiiMxRkHyHAPJdOFrsnkJ2+CSCtMNpQpw3wLrTAl2vINGVgL6LueAodcslAO+gF8o/aB0b2By0k/Dy4fqE39ngHXyJ2wRXHXB/U2vGTL9p69yac00JS2rmO4fHHcAIchxZAoOwbnEr7nghdIgDdN3PhkYZ6cp/197C1bqOsNahqXGuZ0V+F6a7CVIESZR0NsguMlwozEQxvXCPZZY0avqC9HGzOdsqcDUuUOSUJNf7eGwCghTqLCjMTJCn85abCNJwjMHMZXgpMVUOagpebrMK8T2A2MrwUmIkNgQpeDIbWKUmN/ABaKzWzTN7Nf8QpC3ZBAk4WuExYoOKscFkgWjZdoL1PAlXFArUjhGABFZcjQSP9q12LdCSuL4haW4GN1S5q05bRonZtERvxyPbt91u3WmEHa966BAW0/lU0Q23hQutxR9bChfswmit9D2yfdXTus98b95nOSSul/0CXSGA6Ofe9H5xGYYIkDx4mQYWZCT+BUylMsCtMrgpTRaT0ZArTSnaBma3CHAdfwMXsd1xhQlWYieANWEzXLoTC2EIMtpbOtYOgN/hauCEuB55ExgYQx8K/QoBG2lEismMPdGykUSsjhIkQmiHUQdgbpuCqTTAZpmzCVWzAx+BTsAvssgW/zwb8/haYiT+gcwgEn/2kP+N3EADCCRUH8B0HfPywPR/ADtWGjNqH0sBbcGh7+tJWeYlmN5XWDVbER+ND1LdjiWdqJEDiyJmhEum2EFMhEvppGjr6b0wftKk0bwztSih47cn+m5b0GVjfM8wiwzux07vtexdV+ptk7BOZH9/Y59G69YaLA26XKW0KJAp5acD3i/Dd7BWxUBjWpt1vB1OLomD9wRYtfjvE+IfVsbO1SHLyhlnZs0bJna2XCmNRYWbCT5U96+cK012FqSJ6dCiDkV1gvFSYieBNZc8yGJsfkZSqvGf10GzOFOec65Q5vSSFrwECmwjMQtaXZQLZfBU+Z5raIfBwRhrdPegOp64d5OpAbO6urpuPVWlfoQU7Rh+ntQ9X/FULvfGt2r/q6v5aQf6TbPjXusqqWvwleReOA1eNHb+G8e0z5Fl3ysEgEgzSSBxfrhrFtbVGLzUaB/4avgrxkZh7SZqqXZrrGt1dky8wcQVPccQMbvRf4Nzav069+t1M2PX8sf6vRHRsOy8tLx+/t3BE+vApYrcrd//9xrSzaV3xTysrKkKDjgW0yeneC5rWD/y8Z9+CTcuUtWB1v9IVshZdnbpkMQika9FODmBrocJcVmFmwiQQQGFiXWBkyQkjg6oUM4Vor1MgwH0YiwpzPC2K/coDMNJpFWaifwvKRR0oDD1eK6ZaO19vFadj4DMwjULGyxQy3mBLdsoZAcQ1XJeXin1Ae/AY6AJOc9XNmkO9Hl3qLLBSZ3s6CKYrlh5bUZJelk4rntOJ3shOH5GOpim3iitq0hvIC1GeTRc624PYiy2dO6GGapk2fLdtrOaSRKut1bTztDNfH/rwCB5LcPB1o5p4HmwsIRWvLj2Tlfz15opjt375NG9Q3qRrSK49Oem1pPSXx3x9wzFEEFevGrWw35OPnaqflrWh7ZmiucOFjPHTPRA8OM40NKfHqAM79rzeffi4YZnN5TWHumSkZ+G7P62Rl+xv3/6FmF6Hnux4ZFS3zGz0S9kMqdWEUrbG/XAqrU0ma/e4065JY3YNq6uVvif3n3Dy4hLQgnJIiFPfqTBXVJiZsLPCr2EuMLLMYBgvpvlTiFCdAgFUGOmMCjMxMIhyT2sKY2ttsFkUPmugzbeljB8/cto9Y4HE7B7VXgFlAKAC6ZQTRgYzW4hai4bZT4cJTJ70B4NR7B4LQAxKp9o9+wnMTOmgCjMRO4AMvBmMq92TQvi/j3QTWAhX7wSkxJivPAgOIiaNV5BOqc637/Uil4AOJq8ges8Um2EONsWa0k3ZphGmKaYSU5lpr+kt0wcmT+IaBpkoTEis3dcUwvReiIm+AF/K+zQS1lbD1AavtvRDczBLGepcm9r8CAv6Aqf3TjUjCTpLkYnxEVSi0fwbDceQK2fh/uJRk/CX3/+IL0GfSwO3xon6/hn4dp/vLL0jew7Y1uVsH9x8wfaw9eMWbtwq6SfgG/86ewcfhwHVP0BzepyUvztlS9E82aeVvsqY1X560b3U6n1LO2RUPDvnTbpOrL6QyZ9+ivwZyuSPWSeq66TU/TH+6u/kwT0Kf7WWFSgV5rIKMxMOVORhpAuMLDEYxoNDmTyMeGAu2aLCHB/O8Il8EJ/TKszEeCYP21AYWxuDLZxxhEDwfFVMFA+ynI8nSOXPaFOsVLGaNeOowQRAT5aiXs9U2vvvxgd1w6k1S/7ExHq9cBsvpqly9PiXH1y8d/simY/gNZPUHh7m7Cq+1oQZWa52lcDbVa14u4pdqXaVkTCMakpRHlKNLOtD7Koc6H41fnTME+vGDx+F//6lw7CoJ9aNHT2+rmUrGUb4x7cqWQDrA/1lfNm3fUBJCYqshfFGnw1f9LhWZrqNP/FutuFs9z+29FnUBqIhnl4nd3ad2RY67G5uJ/Yoa8FquthaDHHyxm5FFphkN7ZiKswpFWYmHACYNPB3hfmDwTDeGIIYhI5BaOc6qMJMjGOSgMHY/Gk9gfJbrN6HzZfrnM9fmS9QNjXaUitJLDDtv+tj+U/ViTbdx5Km1InWdVozvOkyUd07jje6dOfrRNXnY3TIVehwl9EhUEeejgZ0zYz/IZXBrBaEr6XWN11LXUpLxBU5WthwXdeDnYMVTmxOEgvlDxhRQ6KPbjD35jxE+wgj9SppROAseUfz8768ojfzRcP+XEUJX0Nssaj9zdSxUE/ckNRiVpqq0/WoX5y7OAvXEx8oEwrd1mYLs+lJHPRUjnsF1sKO8YUd9x6o8PCEPaEH7ADdYS+9eyUurMRWX6LykmS3Tyrxp1WfAra3CU0QsZdCQQdiMc3WnJb1yMYQ/ribBGCk+iCBGEoJZQkoj3tmwB8aF1FNlUqM5k7HatW4UVpgmjZoIBeSVG0aadjiM5mZJxb9iv8mEmHxycyMD6fxLTL3xs0vLSkpWVyyQLjT2C0zetjwUTCuzkSkQuHw4YXaphkUuff4CVJ7ffLkTjhG7Z/ZSfLsKcS3dAOhLMuO+Cz7QW9dsC5WJ+Qpx3GSbIOORGytQkpl2dqPoFuZWO+/alXgHwoflooDUIR0geXNOrL8lKCWDKcL2c7yXe/7kWAiAhovms6OUeKVzhs6eM6cwUPnTU6OjkpKiopOlvwGFBcPGFhUNDC6c1JMTDKEyUpPgfi10E/6GxhBAmAlU9qZ3KtpqMtLe8ugXngprh1kk6s1XQwHod/sYd1fsEYmLJk1LOlAXESSVD1i+dDMmLD8VUMz2jM59xIqEn8WOhJL8KvzIMeaweJIqEhy3rOBsWMzKH5dhL/hcCLDJGDQ1GL6siZQo1UwhXV5blbKRfEALMQ73iPw3YQ7MF8Lz/Yqg4fKCaf59AvSIPwczK0CgM2B78Lh0Is/C5WIi+E7F6Zc9MVXoTv0IPhRXNDz5LcjwEkmc0/CJwEARpceDp3q7xJc0FsM/hSDPwX7MXjed/RQbbsuDWa0HYYCiXCDO8WEfRbO0JbYCAc8NzXla9iNjk/iT2HkT+fIGHsBKP4pbEBdhTvAi3CmXfAQol0j+c/MLhw7Z/bYwjmCJX/O7BG9R86YOYLmJ8FWZBUOApl8L4Bsa39ahRoG46EVpvz9Er4CQ15CEXgaXG6Ey+k8Awh8CxVeovBGaIJhRuEeDMFXXvr7b+EgnmvEc2EZXEfgY0CRME2KBAJ9KhDLjqJLjITmV+lhzUXsEGb2/OmogzCIyGQP0Ayk8/H8+31HdllydzbjeAoaycJYVSmq9XIelUkrnSKhVfCJFNCXpaVV2CrCMyer5NvC7G0221Q0w3EAPonw2/SZehK/4AqZOxqUgvsh/wfKsaIjSTlWbDQ7EI2zs/T8YQOAnupMYMhR53bvSHqcDhlskbyrZ6omd+jR5y1cjWeLSa1CZ3KQGGTsLw5om+os9J+wC8ftWPbY1DjfpHlpN/F3G8h/MOxmyvQs34RpSUu3wzM4Dp6BJ9HUV318jnkbYIuPUOWiSv1x2NrgfcJgPFDcrHKRwj97UJHwvdDx4Wf9Ct/T/DYqqlLWyx8A0cz6CFuAyY/qJNS2HjWpPfzJhf9/oseQqvkjL7xw9ewTa3PD02Y/XjT2q6/QuLo60muYW/llcMuTphYFBbmk17DRDugNgBAuWAjPGUA3Dc81d00lIHeRsh2KLYfajLzBeVarnnGeN8950Gz1idShA8XFH+DRHvDFD/EY4bysh6Hr16+fjoKwLEET8mW0H9XwJ7outANRYIsmz95cSznFHnsw726PCmymSZE7s+FqplxJkudpE+aPzpTbHw+GeeStNg3/n82ew3OPzp4zmQTQV4QegaCPpmai+QNnHf+vqyMs/4fqiIfURgwGAG4hOEogRiPTmzd1zjOZnmuXVFO4LIGr5mQsak5mJpzXmKNT8jb/Bbts07oAAAB4AWNgZGAAYen931bF89t8ZZDkYACBIx8E9UD0OZEzun+E/l7lLOKoBHI5GZhAogBOMQvyeAFjYGRg4Ej6e5WBgdPoj9B/I44FQBFUcAcAiWcGPQB4AW2RUxidTQwG52Szv22ztm3btm3btm3btm3bvqvd03y1LuaZrPGGngCA+RkSkWEyhHR6jhTag4r+DBX8n6QKFSOdLKaNrOBb15rftSEZQrtIJGPILCkY6jIjNr+KMd/IZ+QxkhjtjAZGRqNsMCYRGSr/UFW/JbX2oq9Go427QIyP/yWbj8I3/h9G+5+o5tMxWscbE6xdmVp+DqMlJzO1Bclt3mgtwOiPxcbmGI2o7KObO5lzmD+huI7lb9+ATv4Hvv74B6KY4+kdvtQ1FJG4dHCF+dH8hatOQjcCJwPszsXs7l1oo/HJa86vKSgqu4lmdQGjpXxPH/k1PEfj0DaoP7ptc7vQKphrtAksG81RySdb+NnazfUr/vEPiGj+1/jGKCizSSLCLPPvPi8Nn/39X/TWlnbvheT1IympZ/gt9Igueo8S+hcTPspAYdeXBu4c5bQmrYO/f9Z3nM7uM1prdkq7stRw5Sknc2miy+mn35BK0jFGvqGmJLS5k2ls66t99AVzPqpkHKWehigT/PuH+Lhj+E6QRZDDSyRneH+Qg/moscqXIcLLDN5FM5DTN7facniTZzlsY4Bepkvw5x/io7UkeJaDZfAm8lt4kfxGb/MKY6wuI8UbGbxNX9JrV7Pl8BZBDoPpFjjY6+MFVPw4OfndJYbLPNq5I7TxnZn8UVtmhEaSzsgYWK4ZN8gox83b6SL1qCFVKeBGENNNJbXmJLu2Z5RO4RfXnZyuEuVcQZsTn8LB3z0FW2/CPAAAAAAAAAAAAAAALABaANQBSgHaAo4CqgLUAv4DLgNUA2gDgAOaA7IEAgQuBIQFAgVKBbAGGgZQBsgHMAdAB1AHgAeuB94IOgjuCTgJpgn8Cj4KhgrCCygLggueC9QMHgxCDKYM9A1GDYwN6A5MDrIO3g8aD1IPuhAGEEQQfhCkELwQ4BECER4RWBHiEkASkBLuE1IToBQUFFoUhhTKFRIVLhWaFeAWMhaQFuwXLBewGAAYRBh+GOIZPBmSGcwaEBooGmwashqyGtobRBuqHA4ccByaHT4dYB30Ho4emh60HrwfZh98H8ggCiBoIQYhQCGQIboh0CIGIjwihiKSIqwixiLgIzgjSiNcI24jgCOWI6wkIiQuJEAkUiRoJHokjCSeJLQlIiU0JUYlWCVqJXwlkiXEJkImVCZmJngmjiagJu4nVCdmJ3gniiecJ7AnxiiOKJoorCi+KNAo5Cj2KQgpGikwKcop3CnuKgAqEiokKjgqcCrqKvwrDisgKzQrRiukK7gr1CxeLPItGC1YLZQtni2oLcAt2i3uLgYuHi4+Llouci6KLp4u3C9eL3Yv2DAcMKQw9jEcMS4AAAABAAAA3ACXABYAXwAFAAEAAAAAAA4AAAIAAeYAAwABeAF9zANyI2AYBuBnt+YBMsqwjkfpsLY9qmL7Bj1Hb1pbP7+X6HOmy7/uAf8EeJn/GxV4mbvEjL/M3R88Pabfsr0Cbl7mUQdu7am4VNFUEbQp5VpOS8melIyWogt1yyoqMopSkn+kkmIiouKOpNQ15FSUBUWFREWe1ISoWcE378e+mU99WU1NVUlhYZ2nHXKh6sKVrJSQirqMsKKcKyllDSkNYRtWzVu0Zd+iGTEhkXtU0y0IeAFswQOWQgEAAMDZv7Zt27ZtZddTZ+4udYFmBEC5qKCaEjWBQK069Ro0atKsRas27Tp06tKtR68+/QYMGjJsxKgx4yZMmjJtxqw58xYsWrJsxao16zZs2rJtx649+w4cOnLsxKkz5y5cunLtxq079x48evLsxas37z58+vLtx68//0LCIqJi4hKSUtIyshWC4GErEAAAAOAs/3NtI+tluy7Ztm3zZZ6z69yMBuVixBqU50icNMkK1ap48kySXdGy3biVKl+CcYeuFalz786DMo1mTWvy2hsZ3po3Y86yBYuWHHtvzYpVzT64kmnTug0fnTqX6LNPvvjmq+9K/PDLT7/98c9f/wU4EShYkBBhQvUoFSFcpChnLvTZ0qLVtgM72rTr0m1Ch06T4g0ZNvDk+ZMXLo08efk4RnZGDkZOhlQWv1AfH/bSvEwDA0cXEG1kYG7C4lpalM+Rll9apFdcWsBZklGUmgpisZeU54Pp/DwwHwBPQXTqAHgBLc4lXMVQFIDxe5+/Ke4uCXd3KLhLWsWdhvWynugFl7ieRu+dnsb5flD+V44+W03Pqkm96nSsSX3pwfbG8hyVafqKLY53NhRyi8/1/P8l1md6//6SRzsznWXcUiuTXQ3F3NJTfU3V3NRrJp2WrjUzN3sl06/thr54PYV7+IYaQ1++jlly8+AO2iz5W4IT8OEJIqi29NXrGHhwB65DLfxAtSN5HvgQQgRjjiSfQJDDoBz5e4AA3BwJtOVAHgtBBGGeRNsK5DYGd8IvM61XFAA=) format('woff'), + url(../font/Roboto-Medium.woff) format('woff'); +} + + + +/* ---- data/1Hb9rY98TNnA6TYeozJv4w36bqEiBn6x8Y/css/github.css ---- */ + + +/* + +github.com style (c) Vasily Polovnyov + +*/ + +.hljs { + display: block; + overflow-x: auto; + padding: 0.5em; + color: #333; + background: #f8f8f8; + -webkit-text-size-adjust: none; +} + +.hljs-comment, +.diff .hljs-header, +.hljs-javadoc { + color: #998; + font-style: italic; +} + +.hljs-keyword, +.css .rule .hljs-keyword, +.hljs-winutils, +.nginx .hljs-title, +.hljs-subst, +.hljs-request, +.hljs-status { + color: #333; + font-weight: bold; +} + +.hljs-number, +.hljs-hexcolor, +.ruby .hljs-constant { + color: #008080; +} + +.hljs-string, +.hljs-tag .hljs-value, +.hljs-phpdoc, +.hljs-dartdoc, +.tex .hljs-formula { + color: #d14; +} + +.hljs-title, +.hljs-id, +.scss .hljs-preprocessor { + color: #900; + font-weight: bold; +} + +.hljs-list .hljs-keyword, +.hljs-subst { + font-weight: normal; +} + +.hljs-class .hljs-title, +.hljs-type, +.vhdl .hljs-literal, +.tex .hljs-command { + color: #458; + font-weight: bold; +} + +.hljs-tag, +.hljs-tag .hljs-title, +.hljs-rules .hljs-property, +.django .hljs-tag .hljs-keyword { + color: #000080; + font-weight: normal; +} + +.hljs-attribute, +.hljs-variable, +.lisp .hljs-body { + color: #008080; +} + +.hljs-regexp { + color: #009926; +} + +.hljs-symbol, +.ruby .hljs-symbol .hljs-string, +.lisp .hljs-keyword, +.clojure .hljs-keyword, +.scheme .hljs-keyword, +.tex .hljs-special, +.hljs-prompt { + color: #990073; +} + +.hljs-built_in { + color: #0086b3; +} + +.hljs-preprocessor, +.hljs-pragma, +.hljs-pi, +.hljs-doctype, +.hljs-shebang, +.hljs-cdata { + color: #999; + font-weight: bold; +} + +.hljs-deletion { + background: #fdd; +} + +.hljs-addition { + background: #dfd; +} + +.diff .hljs-change { + background: #0086b3; +} + +.hljs-chunk { + color: #aaa; +} + + + +/* ---- data/1Hb9rY98TNnA6TYeozJv4w36bqEiBn6x8Y/css/icons.css ---- */ + + +.icon { display: inline-block; vertical-align: text-bottom; background-repeat: no-repeat; } +.icon-profile { font-size: 6px; top: 0em; -webkit-border-radius: 0.7em 0.7em 0 0; -moz-border-radius: 0.7em 0.7em 0 0; -o-border-radius: 0.7em 0.7em 0 0; -ms-border-radius: 0.7em 0.7em 0 0; border-radius: 0.7em 0.7em 0 0 ; background: #FFFFFF; width: 1.5em; height: 0.7em; position: relative; display: inline-block; margin-right: 4px } +.icon-profile:before { position: absolute; content: ""; top: -1em; left: 0.38em; width: 0.8em; height: 0.85em; -webkit-border-radius: 50%; -moz-border-radius: 50%; -o-border-radius: 50%; -ms-border-radius: 50%; border-radius: 50% ; background: #FFFFFF; } + +.icon-comment { width: 16px; height: 10px; -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; background: #B10DC9; margin-top: 0px; display: inline-block; position: relative; top: -2px; } +.icon-comment:after { left: 9px; border: 2px solid transparent; border-top-color: #B10DC9; border-left-color: #B10DC9; background: transparent; content: ""; display: block; margin-top: 10px; width: 0px; margin-left: 7px; } + +.icon-edit { + width: 16px; height: 16px; background-repeat: no-repeat; background-position: 20px center; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAAOVBMVEUAAAC9w8e9w8e9w8e9w8e/xMi9w8e9w8e+w8e9w8e9w8e9w8e9w8e9w8e9w8e+w8e/xMi9w8e9w8fvY4+KAAAAEnRSTlMASPv3WQbwOTCkt4/psX4YDMWr+RRCAAAAUUlEQVQY06XLORKAMAxDUTs7kA3d/7AYGju0UfffjIgoHkxm0vB5bZyxKHx9eX0FJw0Y4bcXKQ4/CTtS5yqp5GFFOjGpVGl00k1pNDIb3Nv9AHC7BOZC4ZjvAAAAAElFTkSuQmCC+d0ckOwyAMRVGHUOO0gUyd+P8f7WApz4Iki9wFmyOEATrXLZcFp5LrGogPOxKp6zfFf9fZ1/I/cY7YZSS3U6S3XFZJmGBwL+FuJX/F1K0wUUlZyZGlXgXESthTEs4B8fh7xoVUDPGYJnsfkCRarKAgz8cAKbpD6pqDPz3XB8K6HdUEeN9NAAAAAElFTkSuQmCC); +} +.icon-reply { + width: 16px; height: 16px; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQBAMAAADt3eJSAAAAIVBMVEUAAABmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmYs5FxxAAAAC3RSTlMAgBFwYExAMHgoCDJmUTYAAAA3SURBVAjXY8APGGEMQZgAjCEoKBwEEQCCAoiIh6AQVM1kMaguJhGYOSJQjexiUMbiAChDCclCAOHqBBdHpwQTAAAAAElFTkSuQmCC); +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data-default/data.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data-default/data.json new file mode 100644 index 00000000..41392501 --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data-default/data.json @@ -0,0 +1,10 @@ +{ + "title": "MyZeroBlog", + "description": "My ZeroBlog.", + "links": "- [Source code](https://github.com/HelloZeroNet)", + "next_post_id": 1, + "demo": false, + "modified": 1432515193, + "post": [ + ] +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data-default/users/content-default.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data-default/users/content-default.json new file mode 100644 index 00000000..06bfc9cd --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data-default/users/content-default.json @@ -0,0 +1,25 @@ +{ + "files": {}, + "ignore": ".*", + "modified": 1432466966.003, + "signs": { + "1BLogC9LN4oPDcruNz3qo1ysa133E9AGg8": "HChU28lG4MCnAiui6wDAaVCD4QUrgSy4zZ67+MMHidcUJRkLGnO3j4Eb1N0AWQ86nhSBwoOQf08Rha7gRyTDlAk=" + }, + "user_contents": { + "cert_signers": { + "zeroid.bit": [ "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] + }, + "permission_rules": { + ".*": { + "files_allowed": "data.json", + "max_size": 10000 + }, + "bitid/.*@zeroid.bit": { "max_size": 40000 }, + "bitmsg/.*@zeroid.bit": { "max_size": 15000 } + }, + "permissions": { + "banexample@zeroid.bit": false, + "nofish@zeroid.bit": { "max_size": 20000 } + } + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/data.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/data.json new file mode 100644 index 00000000..af289e8f --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/data.json @@ -0,0 +1,244 @@ +{ + "title": "ZeroBlog", + "description": "Demo for decentralized, self publishing blogging platform.", + "links": "- [Source code](https://github.com/HelloZeroNet)\n- [Create new blog](?Post:3:How+to+have+a+blog+like+this)", + "next_post_id": 42, + "demo": false, + "modified": 1433033806, + "post": [ + { + "post_id": 41, + "title": "Changelog: May 31, 2015", + "date_published": 1433033779.604, + "body": " - rev194\n - Ugly OpenSSL memory leak fix\n - Added Docker and Vargant files (thanks to n3r0-ch)\n\nZeroBlog\n - Comment editing, Deleting, Replying added\n\nNew official site: http://zeronet.io/" + }, + { + "post_id": 40, + "title": "Trusted authorization providers", + "date_published": 1432549828.319, + "body": "What is it good for?\n\n - It allows you to have multi-user sites without need of a bot that listen to new user registration requests.\n - You can use the same username across sites\n - The site owner can give you (or revoke) permissions based on your ZeroID username\n\nHow does it works?\n\n - You visit an authorization provider site (eg zeroid.bit)\n - You enter the username you want to register and sent the request to the authorization provider site owner (zeroid supports bitmessage and simple http request).\n - The authorization provider process your request and it he finds everything all right (unique username, other anti-spam methods) he sends you a certificate for the username registration.\n - If a site trust your authorization provider you can post your own content (comments, topics, upvotes, etc.) using this certificate without ever contacting the site owner.\n\nWhat sites currently supports ZeroID?\n\n - You can post comments to ZeroBlog using your ZeroID\n - Later, if everyone is updated to 0.3.0 a new ZeroTalk is also planned that supports ZeroID certificates\n\nWhy is it necessary?\n\n - To have some kind of control over the users of your site. (eg. remove misbehaving users)\n\nOther info\n\n - ZeroID is a standard site, anyone can clone it and have his/her own one\n - You can stop seeding ZeroID site after you got your cert" + }, + { + "post_id": 39, + "title": "Changelog: May 25, 2015", + "date_published": 1432511642.167, + "body": "- Version 0.3.0, rev187\n- Trusted authorization provider support: Easier multi-user sites by allowing site owners to define tusted third-party user certificate signers. (more info about it in the next days)\n- `--publish` option to siteSign to publish automatically after the new files signed.\n- `cryptSign` command line command to sign message using private key.\n- New, more stable OpenSSL layer that also works on OSX.\n- New json table format support.\n- DbCursor SELECT parameters bugfix.\n- Faster multi-threaded peer discovery from trackers.\n- New http trackers added.\n- Wait for dbschema.json file to execute query.\n- Handle json import errors.\n- More compact json writeJson storage command output.\n- Workaround to make non target=_top links work.\n- Cleaner UiWebsocket command router.\n- Notify other local users on local file changes.\n- Option to wait file download before execute query.\n- fileRules, certAdd, certSelect, certSet websocket API commands.\n- Allow more file errors on big sites.\n- On stucked downloads skip worker's current file instead of stopping it.\n- NoParallel parameter bugfix.\n- RateLimit interval bugfix.\n- Updater skips non-writeable files.\n- Try to close OpenSSL dll before update.\n\nZeroBlog:\n- Rewritten to use SQL database\n- Commenting on posts (**Please note: The comment publishing and distribution can be slow until most of the clients is not updated to version 0.3.0**)\n\n![comments](data/img/zeroblog-comments.png)\n\nZeroID\n- Sample Trusted authorization provider site with Bitmessage registration support\n\n![comments](data/img/zeroid.png)" + }, + { + "post_id": 38, + "title": "Status report: Trusted authorization providers", + "date_published": 1431286381.226, + "body": "Currently working on a new feature that allows to create multi-user sites more easily. For example it will allows us to have comments on ZeroBlog (without contacting the site owner).\n\nCurrent status:\n\n - Sign/verification process: 90%\n - Sample trusted authorization provider site: 70%\n - ZeroBlog modifications: 30%\n - Authorization UI enhacements: 10%\n - Total progress: 60%\n \nEta.: 1-2weeks\n\n### Update: May 18, 2015:\n\nThings left:\n - More ZeroBlog modifications on commenting interface\n - Bitmessage support in Sample trusted authorization provider site\n - Test everything on multiple platform/browser and machine\n - Total progress: 80%\n\nIf no major flaw discovered it should be out this week." + }, + { + "post_id": 37, + "title": "Changelog: May 3, 2015", + "date_published": 1430652299.794, + "body": " - rev134\n - Removed ZeroMQ dependencies and support (if you are on pre 0.2.0 version please, upgrade)\n - Save CPU and memory on file requests by streaming content directly to socket without loading to memory and encoding with msgpack.\n - Sites updates without re-download all content.json by querying the modified files from peers.\n - Fix urllib memory leak\n - SiteManager testsuite\n - Fix UiServer security testsuite\n - Announce to tracker on site resume\n\nZeroBoard:\n\n - Only last 100 messages loaded by default\n - Typo fix" + }, + { + "post_id": 36, + "title": "Changelog: Apr 29, 2015", + "date_published": 1430388168.315, + "body": " - rev126\n - You can install the \"127.0.0.1:43110-less\" extension from [Chrome Web Store](https://chrome.google.com/webstore/detail/zeronet-protocol/cpkpdcdljfbnepgfejplkhdnopniieop). (thanks to g0ld3nrati0!)\n - You can disable the use of openssl using `--use_openssl False`\n - OpenSSL disabled on OSX because of possible segfault. You can enable it again using `zeronet.py --use_openssl True`,
please [give your feedback](https://github.com/HelloZeroNet/ZeroNet/issues/94)!\n - Update on non existent file bugfix\n - Save 20% memory using Python slots\n\n![Memory save](data/img/slots_memory.png)" + }, + { + "post_id": 35, + "title": "Changelog: Apr 27, 2015", + "date_published": 1430180561.716, + "body": " - Revision 122\n - 40x faster signature verification by using OpenSSL if available\n - Added OpenSSL benchmark: beat my CPU at http://127.0.0.1:43110/Benchmark :)\n - Fixed UiServer socket memory leak" + }, + { + "post_id": 34, + "title": "Slides about ZeroNet", + "date_published": 1430081791.43, + "body": "Topics:\n - ZeroNet cryptography\n - How site downloading works\n - Site updates\n - Multi-user sites\n - Current status of the project / Future plans\n\n\n\n[Any feedback is welcome!](http://127.0.0.1:43110/Talk.ZeroNetwork.bit/?Topic:18@2/Presentation+about+how+ZeroNet+works) \n\nThanks! :)" + }, + { + "post_id": 33, + "title": "Changelog: Apr 24, 2014", + "date_published": 1429873756.187, + "body": " - Revision 120\n - Batched publishing to avoid update flood: Only send one update in every 7 seconds\n - Protection against update flood by adding update queue: Only allows 1 update in every 10 second for the same file\n - Fix stucked notification icon\n - Fix websocket error when writing to not-owned sites" + }, + { + "post_id": 32, + "title": "Changelog: Apr 20, 2014", + "date_published": 1429572874, + "body": " - Revision 115\n - For faster pageload times allow browser cache on css/js/font files\n - Support for experimental chrome extension that allows to browse zeronet sites using `http://talk.zeronetwork.bit` and/or `http://zero/1Name2NXVi1RDPDgf5617UoW7xA6YrhM9F`\n - Allow to browse memory content in /Stats\n - Peers uses Site's logger to save some memory\n - Give not-that-good peers on initial PEX if required\n - Allows more than one `--ui_restrict` ip address\n - Disable ssl monkey patching to avoid ssl error in Debian Jessie\n - Fixed websocket error when writing not-allowed files\n - Fixed bigsite file not found error\n - Fixed loading screen stays on screen even after index.html loaded\n\nZeroHello:\n\n - Site links converted to 127.0.0.1:43110 -less if using chrome extension\n\n![direct domains](data/img/direct_domains.png)" + }, + { + "post_id": 31, + "title": "Changelog: Apr 17, 2014", + "date_published": 1429319617.201, + "body": " - Revision 101\n - Revision numbering between version\n - Allow passive publishing\n - Start Zeronet when Windows starts option to system tray icon\n - Add peer ping time to publish timeout\n - Passive connected peers always get the updates\n - Pex count bugfix\n - Changed the topright button hamburger utf8 character to more supported one and removed click anim\n - Passive peers only need 3 connection\n - Passive connection store on tracker bugfix\n - Not exits file bugfix\n - You can compare your computer speed (bitcoin crypto, sha512, sqlite access) to mine: http://127.0.0.1:43110/Benchmark :)\n\nZeroTalk:\n\n - Only quote the last message\n - Message height bugfix\n\nZeroHello:\n\n - Changed the burger icon to more supported one\n - Added revision display" + }, + { + "post_id": 30, + "title": "Changelog: Apr 16, 2015", + "date_published": 1429135541.581, + "body": "Apr 15:\n\n - Version 0.2.9\n - To get rid of dead ips only send peers over pex that messaged within 2 hour\n - Only ask peers from 2 sources using pex every 20 min\n - Fixed mysterious notification icon disappearings\n - Mark peers as bad if publish is timed out (5s+)" + }, + { + "post_id": 29, + "title": "Changelog: Apr 15, 2015", + "date_published": 1429060414.445, + "body": " - Sexy system tray icon with statistics instead of ugly console. (sorry, Windows only yet)\n - Total sent/received bytes stats\n - Faster connections and publishing by don't send passive peers using PEX and don't store them on trackers\n\n![Tray icon](data/img/trayicon.png)" + }, + { + "post_id": 28, + "title": "Changelog: Apr 14, 2015", + "date_published": 1428973199.042, + "body": " - Experimental socks proxy support (Tested using Tor)\n - Tracker-less peer exchange between peers\n - Http bittorrent tracker support\n - Option to disable udp connections (udp tracker)\n - Other stability/security fixes\n\nTo use ZeroNet over Tor network start it with `zeronet.py --proxy 127.0.0.1:9050 --disable_udp`\n\nIt's still an experimental feature, there is lot work/fine tuning needed to make it work better and more secure (eg. by supporting hidden service peer addresses to allow connection between Tor clients). \nIn this mode you can only access to sites where there is at least one peer with peer exchange support. (client updated to latest commit)\n\nIf no more bug found i'm going to tag it as 0.2.9 in the next days." + }, + { + "post_id": 27, + "title": "Changelog: Apr 9, 2015", + "date_published": 1428626164.266, + "body": " - Packaged windows dependencies for windows to make it easier to install: [ZeroBundle](https://github.com/HelloZeroNet/ZeroBundle)\n - ZeroName site downloaded at startup, so first .bit domain access is faster.\n - Fixed updater bug. (argh)" + }, + { + "post_id": 26, + "title": "Changelog: Apr 7, 2015", + "date_published": 1428454413.286, + "body": " - Fix for big sites confirmation display\n - Total objects in memory stat\n - Memory optimizations\n - Retry bad files in every 20min\n - Load files to db when executing external siteSign command\n - Fix for endless reconnect bug\n \nZeroTalk:\n \n - Added experimental P2P new bot\n - Bumped size limit to 20k for every user :)\n - Reply button\n\nExperimenting/researching possibilities of i2p/tor support (probably using DHT)\n\nAny help/suggestion/idea greatly welcomed: [github issue](https://github.com/HelloZeroNet/ZeroNet/issues/60)" + }, + { + "post_id": 25, + "title": "Changelog: Apr 2, 2015", + "date_published": 1428022346.555, + "body": " - Better passive mode by making sure to keep 5 active connections\n - Site connection and msgpack unpacker stats\n - No more sha1 hash added to content.json (it was only for backward compatibility with old clients)\n - Keep connection logger object to prevent some exception\n - Retry upnp port opening 3 times\n - Publish received content updates to more peers to make sure the better distribution\n\nZeroTalk: \n\n - Changed edit icon to more clear pencil\n - Single line breaks also breaks the line" + }, + { + "post_id": 24, + "title": "Changelog: Mar 29, 2015", + "date_published": 1427758356.109, + "body": " - Version 0.2.8\n - Namecoin (.bit) domain support!\n - Possible to disable backward compatibility with old version to save some memory\n - Faster content publishing (commenting, posting etc.)\n - Display error on internal server errors\n - Better progress bar\n - Crash and bugfixes\n - Removed coppersurfer tracker (its down atm), added eddie4\n - Sorry, the auto updater broken for this version: please overwrite your current `update.py` file with the [latest one from github](https://raw.githubusercontent.com/HelloZeroNet/ZeroNet/master/update.py), run it and restart ZeroNet.\n - Fixed updater\n\n![domain](data/img/domain.png)\n\nZeroName\n\n - New site for resolving namecoin domains and display registered ones\n\n![ZeroName](data/img/zeroname.png)\nZeroHello\n\n - Automatically links to site's domain names if its specificed in content.json `domain` field\n\n" + }, + { + "post_id": 22, + "title": "Changelog: Mar 23, 2015", + "date_published": 1427159576.994, + "body": " - Version 0.2.7\n - Plugin system: Allows extend ZeroNet without modify the core source\n - Comes with 3 plugin:\n - Multiuser: User login/logout based on BIP32 master seed, generate new master seed on visit (disabled by default to enable it just remove the disabled- from the directory name)\n - Stats: /Stats url moved to separate plugin for demonstration reasons\n - DonationMessage: Puts a little donation link to the bottom of every page (disabled by default)\n - Reworked module import system\n - Lazy user auth_address generatation\n - Allow to send prompt dialog to user from server-side\n - Update script remembers plugins enabled/disabled status\n - Multiline notifications\n - Cookie parser\n\nZeroHello in multiuser mode:\n\n - Logout button\n - Identicon generated based on logined user xpub address\n\n![Multiuser](data/img/multiuser.png)" + }, + { + "post_id": 21, + "title": "Changelog: Mar 19, 2015", + "date_published": 1426818095.915, + "body": " - Version 0.2.6\n - SQL database support that allows easier site development and faster page load times\n - Updated [ZeroFrame API Reference](http://zeronet.readthedocs.org/en/latest/site_development/zeroframe_api_reference/)\n - Added description of new [dbschema.json](http://zeronet.readthedocs.org/en/latest/site_development/dbschema_json/) file\n - SiteStorage class for file operations\n - Incoming connection firstchar errorfix\n - dbRebuild and dbQuery commandline actions\n - [Goals donation page](http://zeronet.readthedocs.org/en/latest/zeronet_development/donate/)\n\nZeroTalk\n\n - Rewritten to use SQL queries (falls back nicely to use json files on older version)" + }, + { + "post_id": 20, + "title": "Changelog: Mar 14, 2015", + "date_published": 1426386779.836, + "body": "\n - Save significant amount of memory by remove unused msgpack unpackers\n - Log unhandled exceptions\n - Connection checker error bugfix\n - Working on database support, you can follow the progress on [reddit](http://www.reddit.com/r/zeronet/comments/2yq7e8/a_json_caching_layer_for_quicker_development_and/)\n\n![memory usage](data/img/memory.png)" + }, + { + "post_id": 19, + "title": "Changelog: Mar 10, 2015", + "date_published": 1426041044.008, + "body": " - Fixed ZeroBoard and ZeroTalk registration: It was down last days, sorry, I haven't tested it after recent modifications, but I promise I will from now :)\n - Working hard on documentations, after trying some possibilities, I chosen readthedocs.org: http://zeronet.readthedocs.org\n - The API reference is now up-to-date, documented demo sites working method and also updated other parts\n\n[Please, tell me what you want to see in the docs, Thanks!](/1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ/?Topic:14@2/New+ZeroNet+documentation)" + }, + { + "post_id": 18, + "title": "Changelog: Mar 8, 2015", + "date_published": 1425865493.306, + "body": " - [Better uPnp Puncher](https://github.com/HelloZeroNet/ZeroNet/blob/master/src/util/UpnpPunch.py), if you have problems with port opening please try this.\n\nZeroTalk: \n - Comment upvoting\n - Topic groups, if you know any other article about ZeroNet please, post [here](/1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ/?Topics:8@2/Articles+about+ZeroNet)" + }, + { + "post_id": 17, + "title": "Changelog: Mar 5, 2015", + "date_published": 1425606285.111, + "body": " - Connection pinging and timeout\n - Request timeout\n - Verify content at signing (size, allowed files)\n - Smarter coffeescript recompile\n - More detailed stats\n\nZeroTalk: \n - Topic upvote\n - Even more source code realign\n\n![ZeroTalk upvote](data/img/zerotalk-upvote.png)" + }, + { + "post_id": 16, + "title": "Changelog: Mar 1, 2015", + "date_published": 1425259087.503, + "body": "ZeroTalk: \n - Reordered source code to allow more more feature in the future\n - Links starting with http://127.0.0.1:43110/ automatically converted to relative links (proxy support)\n - Comment reply (by clicking on comment's creation date)" + }, + { + "post_id": 15, + "title": "Changelog: Feb 25, 2015", + "date_published": 1424913197.035, + "body": " - Version 0.2.5\n - Pure-python upnp port opener (Thanks to sirMackk!)\n - Site download progress bar\n - We are also on [Gitter chat](https://gitter.im/HelloZeroNet/ZeroNet)\n - More detailed connection statistics (ping, buff, idle, delay, sent, received)\n - First char failed bugfix\n - Webebsocket disconnect on slow connection bugfix\n - Faster site update\n\n![Progressbar](data/img/progressbar.png)\n\nZeroTalk: \n\n - Sort after 100ms idle\n - Colored usernames\n - Limit reload rate to 500ms\n\nZeroHello\n\n - [iframe render fps test](/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr/test/render.html) ([more details on ZeroTalk](/1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ/?Topic:7@2/Slow+rendering+in+Chrome))\n" + }, + { + "post_id": 14, + "title": "Changelog: Feb 24, 2015", + "date_published": 1424734437.473, + "body": " - Version 0.2.4\n - New, experimental network code and protocol\n - peerPing and peerGetFile commands\n - Connection share and reuse between sites\n - Don't retry bad file more than 3 times in 20 min\n - Multi-threaded include file download\n - Really shuffle peers before publish\n - Simple internal stats page: http://127.0.0.1:43110/Stats\n - Publish bugfix for sites with more then 10 peers\n\n_If someone on very limited resources its recommended to wait some time until most of the peers is updates to new network code, because the backward compatibility is a little bit tricky and using more memory._" + }, + { + "post_id": 13, + "title": "Changelog: Feb 19, 2015", + "date_published": 1424394659.345, + "body": " - Version 0.2.3\n - One click source code download from github, auto unpack and restart \n - Randomize peers before publish and work start\n - Switched to upnpc-shared.exe it has better virustotal reputation (4/53 vs 19/57)\n\n![Autoupdate](data/img/autoupdate.png)\n\nZeroTalk:\n\n - Topics also sorted by topic creation date\n\n_New content and file changes propagation is a bit broken yet. Already working on better network code that also allows passive content publishing. It will be out in 1-2 weeks._" + }, + { + "post_id": 12, + "title": "Changelog: Feb 16, 2015", + "date_published": 1424134864.167, + "body": "Feb 16: \n - Version 0.2.2\n - LocalStorage support using WrapperAPI\n - Bugfix in user management\n\nZeroTalk: \n - Topics ordered by date of last post\n - Mark updated topics since your last visit\n\n![Mark](data/img/zerotalk-mark.png)" + }, + { + "post_id": 11, + "title": "Changelog: Feb 14, 2015", + "date_published": 1423922572.778, + "body": " - Version 0.2.1\n - Site size limit: Default 10MB, asks permission to store more, test it here: [ZeroNet windows requirement](/1ZeroPYmW4BGwmT6Z54jwPgTWpbKXtTra)\n - Browser open wait until UiServer started\n - Peer numbers stored in sites.json for faster warmup\n - Silent WSGIHandler error\n - siteSetLimit WrapperAPI command\n - Grand ADMIN permission to wrapperframe\n\nZeroHello: \n\n - Site modify time also include sub-file changes (ZeroTalk last comment)\n - Better changetime date format" + }, + { + "post_id": 10, + "title": "Changelog: Feb 11, 2015", + "date_published": 1423701015.643, + "body": "ZeroTalk:\n - Link-type posts\n - You can Edit or Delete your previous Comments and Topics\n - [Uploaded source code to github](https://github.com/HelloZeroNet/ZeroTalk)" + }, + { + "post_id": 9, + "title": "Changelog: Feb 10, 2015", + "date_published": 1423532194.094, + "body": " - Progressive publish timeout based on file size\n - Better tracker error log\n - Viewport support in content.json and ZeroFrame API to allow better mobile device layout\n - Escape ZeroFrame notification messages to avoid js injection\n - Allow select all data in QueryJson\n\nZeroTalk:\n - Display topic's comment number and last comment time (requires ZeroNet today's commits from github)\n - Mobile device optimized layout" + }, + { + "post_id": 8, + "title": "Changelog: Feb 9, 2015", + "date_published": 1423522387.728, + "body": " - Version 0.2.0\n - New bitcoin ECC lib (pybitcointools)\n - Hide notify errors\n - Include support for content.json\n - File permissions (signer address, filesize, allowed filenames)\n - Multisig ready, new, Bitcoincore compatible sign format\n - Faster, multi threaded content publishing\n - Multiuser, ready, BIP32 based site auth using bitcoin address/privatekey\n - Simple json file query language\n - Websocket api fileGet support\n\nZeroTalk: \n - [Decentralized forum demo](/1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ/?Home)\n - Permission request/username registration\n - Everyone has an own file that he able to modify, sign and publish decentralized way, without contacting the site owner\n - Topic creation\n - Per topic commenting\n\n![ZeroTalk screenshot](data/img/zerotalk.png)" + }, + { + "post_id": 7, + "title": "Changelog: Jan 29, 2015", + "date_published": 1422664081.662, + "body": "The default tracker (tracker.pomf.se) is down since yesterday and its resulting some warning messages. To make it disappear please update to latest version from [GitHub](https://github.com/HelloZeroNet/ZeroNet).\n\nZeroNet:\n- Added better tracker error handling\n- Updated alive [trackers list](https://github.com/HelloZeroNet/ZeroNet/blob/master/src/Site/SiteManager.py) (if anyone have more, please [let us know](http://www.reddit.com/r/zeronet/comments/2sgjsp/changelog/co5y07h))\n\nIf you want to stay updated about the project status:
\nWe have created a [@HelloZeronet](https://twitter.com/HelloZeroNet) Twitter account" + }, + { + "post_id": 6, + "title": "Changelog: Jan 27, 2015", + "date_published": 1422394676.432, + "body": "ZeroNet\n* You can use `start.py` to start zeronet and open in browser automatically\n* Send timeout 50sec (workaround for some strange problems until we rewrite the network code without zeromq)\n* Reworked Websocket API to make it unified and allow named and unnamed parameters\n* Reload `content.json` when changed using fileWrite API command\n* Some typo fix\n\nZeroBlog\n* Allow edit post on mainpage\n* Also change blog title in `content.json` when modified using inline editor\n\nZeroHello\n* Update failed warning changed to No peers found when seeding own site." + }, + { + "post_id": 4, + "title": "Changelog: Jan 25, 2015", + "date_published": 1422224700.583, + "body": "ZeroNet\n- Utf-8 site titles fixed\n- Changes in DebugMedia merger to allow faster, node.js based coffeescript compiler\n\nZeroBlog\n- Inline editor rewritten to simple textarea, so copy/paste, undo/redo now working correctly\n- Read more button to folded posts with `---`\n- ZeroBlog running in demo mode, so anyone can try the editing tools\n- Base html tag fixed\n- Markdown cheat-sheet\n- Confirmation if you want to close the browser tab while editing\n\nHow to update your running blog?\n- Backup your `content.json` and `data.json` files\n- Copy the files in the `data/1BLogC9LN4oPDcruNz3qo1ysa133E9AGg8` directory to your site.\n" + }, + { + "post_id": 3, + "title": "How to have a blog like this", + "date_published": 1422140400, + "body": "* Stop ZeroNet\n* Create a new site using `python zeronet.py siteCreate` command\n* Copy all file from **data/1BLogC9LN4oPDcruNz3qo1ysa133E9AGg8** to **data/[Your new site address displayed when executed siteCreate]** directory\n* Delete **data** directory and rename **data-default** to **data** to get a clean, empty site\n* Rename **data/users/content-default.json** file to **data/users/content.json**\n* Execute `zeronet.py siteSign [yoursiteaddress] --inner_path data/users/content.json` to sign commenting rules\n* Start ZeroNet\n* Add/Modify content\n* Click on the `Sign & Publish new content` button\n* Congratulations! Your site is ready to access.\n\n_Note: You have to start commands with `..\\python\\python zeronet.py...` if you downloaded ZeroBundle package_" + }, + { + "post_id": 2, + "title": "Changelog: Jan 24, 2015", + "date_published": 1422105774.057, + "body": "* Version 0.1.6\n* Only serve .html files with wrapper frame\n* Http parameter support in url\n* Customizable background-color for wrapper in content.json\n* New Websocket API commands (only allowed on own sites):\n - fileWrite: Modify site's files in hdd from javascript\n - sitePublish: Sign new content and Publish to peers\n* Prompt value support in ZeroFrame (used for prompting privatekey for publishing in ZeroBlog)\n\n---\n\n## Previous changes:\n\n### Jan 20, 2014\n- Version 0.1.5\n- Detect computer wakeup from sleep and acts as startup (check open port, site changes)\n- Announce interval changed from 10min to 20min\n- Delete site files command support\n- Stop unfinished downloads on pause, delete\n- Confirm dialog support to WrapperApi\n\nZeroHello\n- Site Delete menuitem\n- Browser back button doesn't jumps to top\n\n### Jan 19, 2014:\n- Version 0.1.4\n- WIF compatible new private addresses\n- Proper bitcoin address verification, vanity address support: http://127.0.0.1:43110/1ZEro9ZwiZeEveFhcnubFLiN3v7tDL4bz\n- No hash error on worker kill\n- Have you secured your private key? confirmation\n\n### Jan 18, 2014:\n- Version 0.1.3\n- content.json hashing changed from sha1 to sha512 (trimmed to 256bits) for better security, keep hasing to sha1 for backward compatiblility yet\n- Fixed fileserver_port argument parsing\n- Try to ping peer before asking any command if no communication for 20min\n- Ping timeout / retry\n- Reduce websocket bw usage\n- Separate wrapper_key for websocket auth and auth_key to identify user\n- Removed unnecessary from wrapper iframe url\n\nZeroHello:\n- Compatiblilty with 0.1.3 websocket changes while maintaining backward compatibility\n- Better error report on file update fail\n\nZeroBoard:\n- Support for sha512 hashed auth_key, but keeping md5 key support for older versions yet\n\n### Jan 17, 2014:\n- Version 0.1.2\n- Better error message logging\n- Kill workers on download done\n- Retry on socket error\n- Timestamping console messages\n\n### Jan 16:\n- Version to 0.1.1\n- Version info to websocket api\n- Add publisher's zeronet version to content.json\n- Still chasing network publish problems, added more debug info\n\nZeroHello:\n- Your and the latest ZeroNet version added to top right corner (please update if you dont see it)\n" + }, + { + "post_id": 1, + "title": "ZeroBlog features", + "date_published": 1422105061, + "body": "Initial version (Jan 24, 2014):\n\n* Site avatar generated by site address\n* Distraction-free inline edit: Post title, date, body, Site title, description, links\n* Post format using [markdown](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet)\n* Code block [syntax highlight](#code-highlight-demos) using [highlight.js](https://highlightjs.org/)\n* Create & Delete post\n* Sign & Publish from web\n* Fold blog post: Content after first `---` won't appear at listing\n* Shareable, friendly post urls\n\n\nTodo:\n\n* ~~Better content editor (contenteditable seemed like a good idea, but tricky support of copy/paste makes it more pain than gain)~~\n* Image upload to post & blog avatar\n* Paging\n* Searching\n* ~~Quick cheat-sheet using markdown~~\n\n---\n\n## Code highlight demos\n### Server-side site publishing (UiWebsocket.py):\n```py\ndef actionSitePublish(self, to, params):\n\tsite = self.site\n\tif not site.settings[\"own\"]: return self.response(to, \"Forbidden, you can only modify your own sites\")\n\n\t# Signing\n\tsite.loadContent(True) # Reload content.json, ignore errors to make it up-to-date\n\tsigned = site.signContent(params[0]) # Sign using private key sent by user\n\tif signed:\n\t\tself.cmd(\"notification\", [\"done\", \"Private key correct, site signed!\", 5000]) # Display message for 5 sec\n\telse:\n\t\tself.cmd(\"notification\", [\"error\", \"Site sign failed: invalid private key.\"])\n\t\tself.response(to, \"Site sign failed\")\n\t\treturn\n\tsite.loadContent(True) # Load new content.json, ignore errors\n\n\t# Publishing\n\tif not site.settings[\"serving\"]: # Enable site if paused\n\t\tsite.settings[\"serving\"] = True\n\t\tsite.saveSettings()\n\t\tsite.announce()\n\n\tpublished = site.publish(5) # Publish to 5 peer\n\n\tif published>0: # Successfuly published\n\t\tself.cmd(\"notification\", [\"done\", \"Site published to %s peers.\" % published, 5000])\n\t\tself.response(to, \"ok\")\n\t\tsite.updateWebsocket() # Send updated site data to local websocket clients\n\telse:\n\t\tif len(site.peers) == 0:\n\t\t\tself.cmd(\"notification\", [\"info\", \"No peers found, but your site is ready to access.\"])\n\t\t\tself.response(to, \"No peers found, but your site is ready to access.\")\n\t\telse:\n\t\t\tself.cmd(\"notification\", [\"error\", \"Site publish failed.\"])\n\t\t\tself.response(to, \"Site publish failed.\")\n```\n\n\n### Client-side site publish (ZeroBlog.coffee)\n```coffee\n# Sign and Publish site\npublish: =>\n\tif not @server_info.ip_external # No port open\n\t\t@cmd \"wrapperNotification\", [\"error\", \"To publish the site please open port #{@server_info.fileserver_port} on your router\"]\n\t\treturn false\n\t@cmd \"wrapperPrompt\", [\"Enter your private key:\", \"password\"], (privatekey) => # Prompt the private key\n\t\t$(\".publishbar .button\").addClass(\"loading\")\n\t\t@cmd \"sitePublish\", [privatekey], (res) =>\n\t\t\t$(\".publishbar .button\").removeClass(\"loading\")\n\t\t\t@log \"Publish result:\", res\n\n\treturn false # Ignore link default event\n```\n\n" + } + ] +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/autoupdate.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/autoupdate.png new file mode 100644 index 00000000..7fa439cc Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/autoupdate.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/direct_domains.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/direct_domains.png new file mode 100644 index 00000000..a4eab663 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/direct_domains.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/domain.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/domain.png new file mode 100644 index 00000000..32369cbe Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/domain.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/memory.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/memory.png new file mode 100644 index 00000000..81711e87 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/memory.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/multiuser.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/multiuser.png new file mode 100644 index 00000000..3253ec9e Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/multiuser.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/progressbar.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/progressbar.png new file mode 100644 index 00000000..7eb921a5 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/progressbar.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/slides.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/slides.png new file mode 100644 index 00000000..f8a174f8 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/slides.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/slots_memory.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/slots_memory.png new file mode 100644 index 00000000..fa23d0b5 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/slots_memory.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/trayicon.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/trayicon.png new file mode 100644 index 00000000..f622557b Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/trayicon.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroblog-comments.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroblog-comments.png new file mode 100644 index 00000000..34d3eac6 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroblog-comments.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroid.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroid.png new file mode 100644 index 00000000..014a001b Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroid.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroname.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroname.png new file mode 100644 index 00000000..95cc8fad Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zeroname.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk-mark.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk-mark.png new file mode 100644 index 00000000..c8042ed5 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk-mark.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk-upvote.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk-upvote.png new file mode 100644 index 00000000..b0a7d248 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk-upvote.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk.png b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk.png new file mode 100644 index 00000000..3e2cb5c6 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/img/zerotalk.png differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/optional.txt b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/optional.txt new file mode 100644 index 00000000..3462721f --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/optional.txt @@ -0,0 +1 @@ +hello! \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/test_include/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/test_include/content.json new file mode 100644 index 00000000..814afdbf --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/test_include/content.json @@ -0,0 +1,14 @@ +{ + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "files": { + "data.json": { + "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", + "size": 505 + } + }, + "inner_path": "data/test_include/content.json", + "modified": 1470340816.513, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "GxF2ZD0DaMx+CuxafnnRx+IkWTrXubcmTHaJIPyemFpzCvbSo6DyjstN8T3qngFhYIZI/MkcG4ogStG0PLv6p3w=" + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/test_include/data.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/test_include/data.json new file mode 100644 index 00000000..add2a24b --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/test_include/data.json @@ -0,0 +1,37 @@ +{ + "next_topic_id": 1, + "topics": [], + "next_message_id": 5, + "comments": { + "1@2": [ + { + "comment_id": 1, + "body": "New user test!", + "added": 1423442049 + }, + { + "comment_id": 2, + "body": "test 321", + "added": 1423531445 + }, + { + "comment_id": 3, + "body": "0.2.4 test.", + "added": 1424133003 + } + ] + }, + "topic_votes": { + "1@2": 1, + "1@6": 1, + "1@69": 1, + "607@69": 1 + }, + "comment_votes": { + "35@2": 1, + "7@64": 1, + "8@64": 1, + "50@2": 1, + "13@77": 1 + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json new file mode 100644 index 00000000..bb24e26b --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json @@ -0,0 +1,15 @@ +{ + "cert_auth_type": "web", + "cert_sign": "G4YB7y749GI6mJboyI7cNNfyMwOS0rcVXLmgq8qmCC4TCaRqup3TGWm8hzeru7+B5iXhq19Ruz286bNVKgNbnwU=", + "cert_user_id": "newzeroid@zeroid.bit", + "files": { + "data.json": { + "sha512": "2378ef20379f1db0c3e2a803bfbfda2b68515968b7e311ccc604406168969d34", + "size": 161 + } + }, + "modified": 1432554679.913, + "signs": { + "1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q": "GzX/Ht6ms1dOnqB3kVENvDnxpH+mqA0Zlg3hWy0iwgxpyxWcA4zgmwxcEH41BN9RrvCaxgSd2m1SG1/8qbQPzDY=" + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/data.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/data.json new file mode 100644 index 00000000..52acca16 --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/data.json @@ -0,0 +1,12 @@ +{ + "next_comment_id": 2, + "comment": [ + { + "comment_id": 1, + "body": "Test me!", + "post_id": 40, + "date_added": 1432554679 + } + ], + "comment_vote": {} +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json new file mode 100644 index 00000000..67aaf584 --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json @@ -0,0 +1,24 @@ +{ + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "cert_auth_type": "web", + "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", + "cert_user_id": "toruser@zeroid.bit", + "files": { + "data.json": { + "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", + "size": 168 + } + }, + "files_optional": { + "peanut-butter-jelly-time.gif": { + "sha512": "a238fd27bda2a06f07f9f246954b34dcf82e6472aebdecc2c5dc1f01a50721ef", + "size": 1606 + } + }, + "inner_path": "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json", + "modified": 1470340817.676, + "optional": ".*\\.(jpg|png|gif)", + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G6UOG3ne1hVe3mDGXHnWX8A1vKzH0XHD6LGMsshvNFVXGn003IFNLUL9dlb3XXJf3tyJGZncvGobzNpwBib08QY=" + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/data.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/data.json new file mode 100644 index 00000000..1a7e9ee0 --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/data.json @@ -0,0 +1,12 @@ +{ + "next_comment_id": 2, + "comment": [ + { + "comment_id": 1, + "body": "hello from Tor!", + "post_id": 38, + "date_added": 1432491109 + } + ], + "comment_vote": {} +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif new file mode 100644 index 00000000..54c69d1c Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json new file mode 100644 index 00000000..7436b6da --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json @@ -0,0 +1,17 @@ +{ + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "cert_auth_type": "web", + "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", + "cert_user_id": "toruser@zeroid.bit", + "files": { + "data.json": { + "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", + "size": 168 + } + }, + "inner_path": "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", + "modified": 1470340818.389, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G6oCzql6KWKAq2aSmZ1pm4SqvwL3e3LRdWxsvILrDc6VWpGZmVgbNn5qW18bA7fewhtA/oKc5+yYjGlTLLOWrB4=" + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/data.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/data.json new file mode 100644 index 00000000..1a7e9ee0 --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/data.json @@ -0,0 +1,12 @@ +{ + "next_comment_id": 2, + "comment": [ + { + "comment_id": 1, + "body": "hello from Tor!", + "post_id": 38, + "date_added": 1432491109 + } + ], + "comment_vote": {} +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/content.json new file mode 100644 index 00000000..8c71b84a --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/data/users/content.json @@ -0,0 +1,30 @@ +{ + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "files": {}, + "ignore": ".*", + "inner_path": "data/users/content.json", + "modified": 1470340815.228, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G25hsrlyTOy8PHKuovKDRC7puoBj/OLIZ3U4OJ01izkhE1BBQ+TOgxX96+HXoZGme2/P4IdEnYjc1rqIZ6O+nFk=" + }, + "user_contents": { + "cert_signers": { + "zeroid.bit": [ "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] + }, + "permission_rules": { + ".*": { + "files_allowed": "data.json", + "files_allowed_optional": ".*\\.(png|jpg|gif)", + "max_size": 10000, + "max_size_optional": 10000000, + "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ] + }, + "bitid/.*@zeroid.bit": { "max_size": 40000 }, + "bitmsg/.*@zeroid.bit": { "max_size": 15000 } + }, + "permissions": { + "bad@zeroid.bit": false, + "nofish@zeroid.bit": { "max_size": 100000 } + } + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/dbschema.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/dbschema.json new file mode 100644 index 00000000..3d1cdd7a --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/dbschema.json @@ -0,0 +1,54 @@ +{ + "db_name": "ZeroBlog", + "db_file": "data/zeroblog.db", + "version": 2, + "maps": { + "users/.+/data.json": { + "to_table": [ + "comment", + {"node": "comment_vote", "table": "comment_vote", "key_col": "comment_uri", "val_col": "vote"} + ] + }, + "users/.+/content.json": { + "to_keyvalue": [ "cert_user_id" ] + }, + "data.json": { + "to_table": [ "post" ], + "to_keyvalue": [ "title", "description", "links", "next_post_id", "demo", "modified" ] + } + + }, + "tables": { + "comment": { + "cols": [ + ["comment_id", "INTEGER"], + ["post_id", "INTEGER"], + ["body", "TEXT"], + ["date_added", "INTEGER"], + ["json_id", "INTEGER REFERENCES json (json_id)"] + ], + "indexes": ["CREATE UNIQUE INDEX comment_key ON comment(json_id, comment_id)", "CREATE INDEX comment_post_id ON comment(post_id)"], + "schema_changed": 1426195823 + }, + "comment_vote": { + "cols": [ + ["comment_uri", "TEXT"], + ["vote", "INTEGER"], + ["json_id", "INTEGER REFERENCES json (json_id)"] + ], + "indexes": ["CREATE INDEX comment_vote_comment_uri ON comment_vote(comment_uri)", "CREATE INDEX comment_vote_json_id ON comment_vote(json_id)"], + "schema_changed": 1426195822 + }, + "post": { + "cols": [ + ["post_id", "INTEGER"], + ["title", "TEXT"], + ["body", "TEXT"], + ["date_published", "INTEGER"], + ["json_id", "INTEGER REFERENCES json (json_id)"] + ], + "indexes": ["CREATE UNIQUE INDEX post_uri ON post(json_id, post_id)", "CREATE INDEX post_id ON post(post_id)"], + "schema_changed": 1426195823 + } + } +} \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/img/loading.gif b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/img/loading.gif new file mode 100644 index 00000000..27d0aa81 Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/img/loading.gif differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/index.html b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/index.html new file mode 100644 index 00000000..9feb328b --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/index.html @@ -0,0 +1,137 @@ + + + + + ZeroBlog Demo + + + + + + + + + + +
+
    +
  • # H1
    • +
  • ## H2
    • +
  • ### H3
    • +
  • _italic_
    • +
  • **bold**
    • +
  • ~~strikethrough~~
    • +
  • - Lists
    • +
  • 1. Numbered lists
    • +
  • [Links](http://www.zeronet.io)
    • +
  • [References][1]
    [1]: Can be used
    • +
  • ![image alt](img/logo.png)
    • +
  • Inline `code`
    • +
  • ```python
    print "Code block"
    ```
    • +
  • > Quotes
    • +
  • --- Horizontal rule
    • +
+ ? Editing: Post:21.body Save Delete Cancel +
+ + + + +
+ Content changed Sign & Publish new content +
+ + + + +
+
+

+

+
+
+
+
+ + + + +
+ + + +
+ Add new post + + +
+

Title

+
+ 21 hours ago · 2 min read + ·
3 comments +
+
Body
+ Read more → +
+ + +
+ + + +
+

Title

+
21 hours ago · 2 min read
+
+ +

0 Comments:

+ +
+
+ Please sign in + ━ + new comment +
+
+
Sign in as... + + Submit comment +
+
+
+
+
+
+
+ + +
+ +
+
+ user_name + + ━ + 1 day ago +
Reply +
+
Body
+
+ +
+
+ + + + +
+ + + +
+ + + + + + diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/js/all.js b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/js/all.js new file mode 100644 index 00000000..99eb0c2e --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT-original/js/all.js @@ -0,0 +1,1892 @@ + + +/* ---- data/1BLogC9LN4oPDcruNz3qo1ysa133E9AGg8/js/lib/00-jquery.min.js ---- */ + + +/*! jQuery v2.1.3 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l=a.document,m="2.1.3",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return n.each(this,a,b)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(n.isPlainObject(d)||(e=n.isArray(d)))?(e?(e=!1,f=c&&n.isArray(c)?c:[]):f=c&&n.isPlainObject(c)?c:{},g[b]=n.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray,isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){return!n.isArray(a)&&a-parseFloat(a)+1>=0},isPlainObject:function(a){return"object"!==n.type(a)||a.nodeType||n.isWindow(a)?!1:a.constructor&&!j.call(a.constructor.prototype,"isPrototypeOf")?!1:!0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(a){var b,c=eval;a=n.trim(a),a&&(1===a.indexOf("use strict")?(b=l.createElement("script"),b.text=a,l.head.appendChild(b).parentNode.removeChild(b)):c(a))},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){return null==b?-1:g.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;c>d;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=s(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(c=a[b],b=a,a=c),n.isFunction(a)?(e=d.call(arguments,2),f=function(){return a.apply(b||this,e.concat(d.call(arguments)))},f.guid=a.guid=a.guid||n.guid++,f):void 0},now:Date.now,support:k}),n.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=hb(),z=hb(),A=hb(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N=M.replace("w","w#"),O="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+N+"))|)"+L+"*\\]",P=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+O+")*)|.*)\\)|)",Q=new RegExp(L+"+","g"),R=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),S=new RegExp("^"+L+"*,"+L+"*"),T=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),U=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),V=new RegExp(P),W=new RegExp("^"+N+"$"),X={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M.replace("w","w*")+")"),ATTR:new RegExp("^"+O),PSEUDO:new RegExp("^"+P),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},eb=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(fb){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function gb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],k=b.nodeType,"string"!=typeof a||!a||1!==k&&9!==k&&11!==k)return d;if(!e&&p){if(11!==k&&(f=_.exec(a)))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return H.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName)return H.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=1!==k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+rb(o[l]);w=ab.test(a)&&pb(b.parentNode)||b,x=o.join(",")}if(x)try{return H.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function hb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ib(a){return a[u]=!0,a}function jb(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function kb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function lb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function nb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function ob(a){return ib(function(b){return b=+b,ib(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function pb(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=gb.support={},f=gb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=gb.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=g.documentElement,e=g.defaultView,e&&e!==e.top&&(e.addEventListener?e.addEventListener("unload",eb,!1):e.attachEvent&&e.attachEvent("onunload",eb)),p=!f(g),c.attributes=jb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=jb(function(a){return a.appendChild(g.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(g.getElementsByClassName),c.getById=jb(function(a){return o.appendChild(a).id=u,!g.getElementsByName||!g.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(g.querySelectorAll))&&(jb(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),jb(function(a){var b=g.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&jb(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",P)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===g||a.ownerDocument===v&&t(v,a)?-1:b===g||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,h=[a],i=[b];if(!e||!f)return a===g?-1:b===g?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return lb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?lb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},g):n},gb.matches=function(a,b){return gb(a,null,null,b)},gb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return gb(b,n,null,[a]).length>0},gb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},gb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},gb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},gb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=gb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=gb.selectors={cacheLength:50,createPseudo:ib,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||gb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&gb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=gb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(Q," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||gb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ib(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ib(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?ib(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ib(function(a){return function(b){return gb(a,b).length>0}}),contains:ib(function(a){return a=a.replace(cb,db),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ib(function(a){return W.test(a||"")||gb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:ob(function(){return[0]}),last:ob(function(a,b){return[b-1]}),eq:ob(function(a,b,c){return[0>c?c+b:c]}),even:ob(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:ob(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:ob(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:ob(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function sb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function tb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ub(a,b,c){for(var d=0,e=b.length;e>d;d++)gb(a,b[d],c);return c}function vb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function wb(a,b,c,d,e,f){return d&&!d[u]&&(d=wb(d)),e&&!e[u]&&(e=wb(e,f)),ib(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ub(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:vb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=vb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=vb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function xb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=sb(function(a){return a===b},h,!0),l=sb(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[sb(tb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return wb(i>1&&tb(m),i>1&&rb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&xb(a.slice(i,e)),f>e&&xb(a=a.slice(e)),f>e&&rb(a))}m.push(c)}return tb(m)}function yb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=F.call(i));s=vb(s)}H.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&gb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?ib(f):f}return h=gb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=xb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,yb(e,d)),f.selector=a}return f},i=gb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&pb(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&rb(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&pb(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=jb(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),jb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||kb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&jb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||kb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),jb(function(a){return null==a.getAttribute("disabled")})||kb(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),gb}(a);n.find=t,n.expr=t.selectors,n.expr[":"]=n.expr.pseudos,n.unique=t.uniqueSort,n.text=t.getText,n.isXMLDoc=t.isXML,n.contains=t.contains;var u=n.expr.match.needsContext,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^.[^:#\[\.,]*$/;function x(a,b,c){if(n.isFunction(b))return n.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return n.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(w.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return g.call(b,a)>=0!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=this.length,d=[],e=this;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;c>b;b++)if(n.contains(e[b],this))return!0}));for(b=0;c>b;b++)n.find(a,e[b],d);return d=this.pushStack(c>1?n.unique(d):d),d.selector=this.selector?this.selector+" "+a:a,d},filter:function(a){return this.pushStack(x(this,a||[],!1))},not:function(a){return this.pushStack(x(this,a||[],!0))},is:function(a){return!!x(this,"string"==typeof a&&u.test(a)?n(a):a||[],!1).length}});var y,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=n.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||y).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof n?b[0]:b,n.merge(this,n.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:l,!0)),v.test(c[1])&&n.isPlainObject(b))for(c in b)n.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}return d=l.getElementById(c[2]),d&&d.parentNode&&(this.length=1,this[0]=d),this.context=l,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):n.isFunction(a)?"undefined"!=typeof y.ready?y.ready(a):a(n):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),n.makeArray(a,this))};A.prototype=n.fn,y=n(l);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};n.extend({dir:function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&n(a).is(c))break;d.push(a)}return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),n.fn.extend({has:function(a){var b=n(a,this),c=b.length;return this.filter(function(){for(var a=0;c>a;a++)if(n.contains(this,b[a]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=u.test(a)||"string"!=typeof a?n(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&n.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?n.unique(f):f)},index:function(a){return a?"string"==typeof a?g.call(n(a),this[0]):g.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(n.unique(n.merge(this.get(),n(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){while((a=a[b])&&1!==a.nodeType);return a}n.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return n.dir(a,"parentNode")},parentsUntil:function(a,b,c){return n.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return n.dir(a,"nextSibling")},prevAll:function(a){return n.dir(a,"previousSibling")},nextUntil:function(a,b,c){return n.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return n.dir(a,"previousSibling",c)},siblings:function(a){return n.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return n.sibling(a.firstChild)},contents:function(a){return a.contentDocument||n.merge([],a.childNodes)}},function(a,b){n.fn[a]=function(c,d){var e=n.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=n.filter(d,e)),this.length>1&&(C[a]||n.unique(e),B.test(a)&&e.reverse()),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return n.each(a.match(E)||[],function(a,c){b[c]=!0}),b}n.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):n.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(b=a.memory&&l,c=!0,g=e||0,e=0,f=h.length,d=!0;h&&f>g;g++)if(h[g].apply(l[0],l[1])===!1&&a.stopOnFalse){b=!1;break}d=!1,h&&(i?i.length&&j(i.shift()):b?h=[]:k.disable())},k={add:function(){if(h){var c=h.length;!function g(b){n.each(b,function(b,c){var d=n.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&g(c)})}(arguments),d?f=h.length:b&&(e=c,j(b))}return this},remove:function(){return h&&n.each(arguments,function(a,b){var c;while((c=n.inArray(b,h,c))>-1)h.splice(c,1),d&&(f>=c&&f--,g>=c&&g--)}),this},has:function(a){return a?n.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],f=0,this},disable:function(){return h=i=b=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,b||k.disable(),this},locked:function(){return!i},fireWith:function(a,b){return!h||c&&!i||(b=b||[],b=[a,b.slice?b.slice():b],d?i.push(b):j(b)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!c}};return k},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&n.isFunction(a.promise)?e:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){(a===!0?--n.readyWait:n.isReady)||(n.isReady=!0,a!==!0&&--n.readyWait>0||(H.resolveWith(l,[n]),n.fn.triggerHandler&&(n(l).triggerHandler("ready"),n(l).off("ready"))))}});function I(){l.removeEventListener("DOMContentLoaded",I,!1),a.removeEventListener("load",I,!1),n.ready()}n.ready.promise=function(b){return H||(H=n.Deferred(),"complete"===l.readyState?setTimeout(n.ready):(l.addEventListener("DOMContentLoaded",I,!1),a.addEventListener("load",I,!1))),H.promise(b)},n.ready.promise();var J=n.access=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===n.type(c)){e=!0;for(h in c)n.access(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,n.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(n(a),c)})),b))for(;i>h;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f};n.acceptData=function(a){return 1===a.nodeType||9===a.nodeType||!+a.nodeType};function K(){Object.defineProperty(this.cache={},0,{get:function(){return{}}}),this.expando=n.expando+K.uid++}K.uid=1,K.accepts=n.acceptData,K.prototype={key:function(a){if(!K.accepts(a))return 0;var b={},c=a[this.expando];if(!c){c=K.uid++;try{b[this.expando]={value:c},Object.defineProperties(a,b)}catch(d){b[this.expando]=c,n.extend(a,b)}}return this.cache[c]||(this.cache[c]={}),c},set:function(a,b,c){var d,e=this.key(a),f=this.cache[e];if("string"==typeof b)f[b]=c;else if(n.isEmptyObject(f))n.extend(this.cache[e],b);else for(d in b)f[d]=b[d];return f},get:function(a,b){var c=this.cache[this.key(a)];return void 0===b?c:c[b]},access:function(a,b,c){var d;return void 0===b||b&&"string"==typeof b&&void 0===c?(d=this.get(a,b),void 0!==d?d:this.get(a,n.camelCase(b))):(this.set(a,b,c),void 0!==c?c:b)},remove:function(a,b){var c,d,e,f=this.key(a),g=this.cache[f];if(void 0===b)this.cache[f]={};else{n.isArray(b)?d=b.concat(b.map(n.camelCase)):(e=n.camelCase(b),b in g?d=[b,e]:(d=e,d=d in g?[d]:d.match(E)||[])),c=d.length;while(c--)delete g[d[c]]}},hasData:function(a){return!n.isEmptyObject(this.cache[a[this.expando]]||{})},discard:function(a){a[this.expando]&&delete this.cache[a[this.expando]]}};var L=new K,M=new K,N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){var d;if(void 0===c&&1===a.nodeType)if(d="data-"+b.replace(O,"-$1").toLowerCase(),c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}M.set(a,b,c)}else c=void 0;return c}n.extend({hasData:function(a){return M.hasData(a)||L.hasData(a)},data:function(a,b,c){return M.access(a,b,c) +},removeData:function(a,b){M.remove(a,b)},_data:function(a,b,c){return L.access(a,b,c)},_removeData:function(a,b){L.remove(a,b)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=M.get(f),1===f.nodeType&&!L.get(f,"hasDataAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d])));L.set(f,"hasDataAttrs",!0)}return e}return"object"==typeof a?this.each(function(){M.set(this,a)}):J(this,function(b){var c,d=n.camelCase(a);if(f&&void 0===b){if(c=M.get(f,a),void 0!==c)return c;if(c=M.get(f,d),void 0!==c)return c;if(c=P(f,d,void 0),void 0!==c)return c}else this.each(function(){var c=M.get(this,d);M.set(this,d,b),-1!==a.indexOf("-")&&void 0!==c&&M.set(this,a,b)})},null,b,arguments.length>1,null,!0)},removeData:function(a){return this.each(function(){M.remove(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=L.get(a,b),c&&(!d||n.isArray(c)?d=L.access(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return L.get(a,c)||L.access(a,c,{empty:n.Callbacks("once memory").add(function(){L.remove(a,[b+"queue",c])})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthx",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var U="undefined";k.focusinBubbles="onfocusin"in a;var V=/^key/,W=/^(?:mouse|pointer|contextmenu)|click/,X=/^(?:focusinfocus|focusoutblur)$/,Y=/^([^.]*)(?:\.(.+)|)$/;function Z(){return!0}function $(){return!1}function _(){try{return l.activeElement}catch(a){}}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=L.get(a);if(r){c.handler&&(f=c,c=f.handler,e=f.selector),c.guid||(c.guid=n.guid++),(i=r.events)||(i=r.events={}),(g=r.handle)||(g=r.handle=function(b){return typeof n!==U&&n.event.triggered!==b.type?n.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(E)||[""],j=b.length;while(j--)h=Y.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o&&(l=n.event.special[o]||{},o=(e?l.delegateType:l.bindType)||o,l=n.event.special[o]||{},k=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},f),(m=i[o])||(m=i[o]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,p,g)!==!1||a.addEventListener&&a.addEventListener(o,g,!1)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),n.event.global[o]=!0)}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=L.hasData(a)&&L.get(a);if(r&&(i=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=Y.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=i[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&q!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete i[o])}else for(o in i)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(i)&&(delete r.handle,L.remove(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,m,o,p=[d||l],q=j.call(b,"type")?b.type:b,r=j.call(b,"namespace")?b.namespace.split("."):[];if(g=h=d=d||l,3!==d.nodeType&&8!==d.nodeType&&!X.test(q+n.event.triggered)&&(q.indexOf(".")>=0&&(r=q.split("."),q=r.shift(),r.sort()),k=q.indexOf(":")<0&&"on"+q,b=b[n.expando]?b:new n.Event(q,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=r.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+r.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:n.makeArray(c,[b]),o=n.event.special[q]||{},e||!o.trigger||o.trigger.apply(d,c)!==!1)){if(!e&&!o.noBubble&&!n.isWindow(d)){for(i=o.delegateType||q,X.test(i+q)||(g=g.parentNode);g;g=g.parentNode)p.push(g),h=g;h===(d.ownerDocument||l)&&p.push(h.defaultView||h.parentWindow||a)}f=0;while((g=p[f++])&&!b.isPropagationStopped())b.type=f>1?i:o.bindType||q,m=(L.get(g,"events")||{})[b.type]&&L.get(g,"handle"),m&&m.apply(g,c),m=k&&g[k],m&&m.apply&&n.acceptData(g)&&(b.result=m.apply(g,c),b.result===!1&&b.preventDefault());return b.type=q,e||b.isDefaultPrevented()||o._default&&o._default.apply(p.pop(),c)!==!1||!n.acceptData(d)||k&&n.isFunction(d[q])&&!n.isWindow(d)&&(h=d[k],h&&(d[k]=null),n.event.triggered=q,d[q](),n.event.triggered=void 0,h&&(d[k]=h)),b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(L.get(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,c=0;while((g=f.handlers[c++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(g.namespace))&&(a.handleObj=g,a.data=g.data,e=((n.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==e&&(a.result=e)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!==this;i=i.parentNode||this)if(i.disabled!==!0||"click"!==a.type){for(d=[],c=0;h>c;c++)f=b[c],e=f.selector+" ",void 0===d[e]&&(d[e]=f.needsContext?n(e,this).index(i)>=0:n.find(e,this,null,[i]).length),d[e]&&d.push(f);d.length&&g.push({elem:i,handlers:d})}return h]*)\/>/gi,bb=/<([\w:]+)/,cb=/<|&#?\w+;/,db=/<(?:script|style|link)/i,eb=/checked\s*(?:[^=]|=\s*.checked.)/i,fb=/^$|\/(?:java|ecma)script/i,gb=/^true\/(.*)/,hb=/^\s*\s*$/g,ib={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};ib.optgroup=ib.option,ib.tbody=ib.tfoot=ib.colgroup=ib.caption=ib.thead,ib.th=ib.td;function jb(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function kb(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function lb(a){var b=gb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function mb(a,b){for(var c=0,d=a.length;d>c;c++)L.set(a[c],"globalEval",!b||L.get(b[c],"globalEval"))}function nb(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(L.hasData(a)&&(f=L.access(a),g=L.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;d>c;c++)n.event.add(b,e,j[e][c])}M.hasData(a)&&(h=M.access(a),i=n.extend({},h),M.set(b,i))}}function ob(a,b){var c=a.getElementsByTagName?a.getElementsByTagName(b||"*"):a.querySelectorAll?a.querySelectorAll(b||"*"):[];return void 0===b||b&&n.nodeName(a,b)?n.merge([a],c):c}function pb(a,b){var c=b.nodeName.toLowerCase();"input"===c&&T.test(a.type)?b.checked=a.checked:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}n.extend({clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=n.contains(a.ownerDocument,a);if(!(k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(g=ob(h),f=ob(a),d=0,e=f.length;e>d;d++)pb(f[d],g[d]);if(b)if(c)for(f=f||ob(a),g=g||ob(h),d=0,e=f.length;e>d;d++)nb(f[d],g[d]);else nb(a,h);return g=ob(h,"script"),g.length>0&&mb(g,!i&&ob(a,"script")),h},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,k=b.createDocumentFragment(),l=[],m=0,o=a.length;o>m;m++)if(e=a[m],e||0===e)if("object"===n.type(e))n.merge(l,e.nodeType?[e]:e);else if(cb.test(e)){f=f||k.appendChild(b.createElement("div")),g=(bb.exec(e)||["",""])[1].toLowerCase(),h=ib[g]||ib._default,f.innerHTML=h[1]+e.replace(ab,"<$1>")+h[2],j=h[0];while(j--)f=f.lastChild;n.merge(l,f.childNodes),f=k.firstChild,f.textContent=""}else l.push(b.createTextNode(e));k.textContent="",m=0;while(e=l[m++])if((!d||-1===n.inArray(e,d))&&(i=n.contains(e.ownerDocument,e),f=ob(k.appendChild(e),"script"),i&&mb(f),c)){j=0;while(e=f[j++])fb.test(e.type||"")&&c.push(e)}return k},cleanData:function(a){for(var b,c,d,e,f=n.event.special,g=0;void 0!==(c=a[g]);g++){if(n.acceptData(c)&&(e=c[L.expando],e&&(b=L.cache[e]))){if(b.events)for(d in b.events)f[d]?n.event.remove(c,d):n.removeEvent(c,d,b.handle);L.cache[e]&&delete L.cache[e]}delete M.cache[c[M.expando]]}}}),n.fn.extend({text:function(a){return J(this,function(a){return void 0===a?n.text(this):this.empty().each(function(){(1===this.nodeType||11===this.nodeType||9===this.nodeType)&&(this.textContent=a)})},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=jb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=jb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?n.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||n.cleanData(ob(c)),c.parentNode&&(b&&n.contains(c.ownerDocument,c)&&mb(ob(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(n.cleanData(ob(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return J(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!db.test(a)&&!ib[(bb.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(ab,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(ob(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,n.cleanData(ob(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,m=this,o=l-1,p=a[0],q=n.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&eb.test(p))return this.each(function(c){var d=m.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(c=n.buildFragment(a,this[0].ownerDocument,!1,this),d=c.firstChild,1===c.childNodes.length&&(c=d),d)){for(f=n.map(ob(c,"script"),kb),g=f.length;l>j;j++)h=c,j!==o&&(h=n.clone(h,!0,!0),g&&n.merge(f,ob(h,"script"))),b.call(this[j],h,j);if(g)for(i=f[f.length-1].ownerDocument,n.map(f,lb),j=0;g>j;j++)h=f[j],fb.test(h.type||"")&&!L.access(h,"globalEval")&&n.contains(i,h)&&(h.src?n._evalUrl&&n._evalUrl(h.src):n.globalEval(h.textContent.replace(hb,"")))}return this}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=[],e=n(a),g=e.length-1,h=0;g>=h;h++)c=h===g?this:this.clone(!0),n(e[h])[b](c),f.apply(d,c.get());return this.pushStack(d)}});var qb,rb={};function sb(b,c){var d,e=n(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:n.css(e[0],"display");return e.detach(),f}function tb(a){var b=l,c=rb[a];return c||(c=sb(a,b),"none"!==c&&c||(qb=(qb||n(" + + + + + + + diff --git a/src/User/User.py b/src/User/User.py new file mode 100644 index 00000000..dbcfc56f --- /dev/null +++ b/src/User/User.py @@ -0,0 +1,176 @@ +import logging +import json +import time +import binascii + +import gevent + +import util +from Crypt import CryptBitcoin +from Plugin import PluginManager +from Config import config +from util import helper +from Debug import Debug + + +@PluginManager.acceptPlugins +class User(object): + def __init__(self, master_address=None, master_seed=None, data={}): + if master_seed: + self.master_seed = master_seed + self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed) + elif master_address: + self.master_address = master_address + self.master_seed = data.get("master_seed") + else: + self.master_seed = CryptBitcoin.newSeed() + self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed) + self.sites = data.get("sites", {}) + self.certs = data.get("certs", {}) + self.settings = data.get("settings", {}) + self.delayed_save_thread = None + + self.log = logging.getLogger("User:%s" % self.master_address) + + # Save to data/users.json + @util.Noparallel(queue=True, ignore_class=True) + def save(self): + s = time.time() + users = json.load(open("%s/users.json" % config.data_dir)) + if self.master_address not in users: + users[self.master_address] = {} # Create if not exist + user_data = users[self.master_address] + if self.master_seed: + user_data["master_seed"] = self.master_seed + user_data["sites"] = self.sites + user_data["certs"] = self.certs + user_data["settings"] = self.settings + helper.atomicWrite("%s/users.json" % config.data_dir, helper.jsonDumps(users).encode("utf8")) + self.log.debug("Saved in %.3fs" % (time.time() - s)) + self.delayed_save_thread = None + + def saveDelayed(self): + if not self.delayed_save_thread: + self.delayed_save_thread = gevent.spawn_later(5, self.save) + + def getAddressAuthIndex(self, address): + return int(binascii.hexlify(address.encode()), 16) + + @util.Noparallel() + def generateAuthAddress(self, address): + s = time.time() + address_id = self.getAddressAuthIndex(address) # Convert site address to int + auth_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, address_id) + self.sites[address] = { + "auth_address": CryptBitcoin.privatekeyToAddress(auth_privatekey), + "auth_privatekey": auth_privatekey + } + self.saveDelayed() + self.log.debug("Added new site: %s in %.3fs" % (address, time.time() - s)) + return self.sites[address] + + # Get user site data + # Return: {"auth_address": "xxx", "auth_privatekey": "xxx"} + def getSiteData(self, address, create=True): + if address not in self.sites: # Generate new BIP32 child key based on site address + if not create: + return {"auth_address": None, "auth_privatekey": None} # Dont create user yet + self.generateAuthAddress(address) + return self.sites[address] + + def deleteSiteData(self, address): + if address in self.sites: + del(self.sites[address]) + self.saveDelayed() + self.log.debug("Deleted site: %s" % address) + + def setSiteSettings(self, address, settings): + site_data = self.getSiteData(address) + site_data["settings"] = settings + self.saveDelayed() + return site_data + + # Get data for a new, unique site + # Return: [site_address, bip32_index, {"auth_address": "xxx", "auth_privatekey": "xxx", "privatekey": "xxx"}] + def getNewSiteData(self): + import random + bip32_index = random.randrange(2 ** 256) % 100000000 + site_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, bip32_index) + site_address = CryptBitcoin.privatekeyToAddress(site_privatekey) + if site_address in self.sites: + raise Exception("Random error: site exist!") + # Save to sites + self.getSiteData(site_address) + self.sites[site_address]["privatekey"] = site_privatekey + self.save() + return site_address, bip32_index, self.sites[site_address] + + # Get BIP32 address from site address + # Return: BIP32 auth address + def getAuthAddress(self, address, create=True): + cert = self.getCert(address) + if cert: + return cert["auth_address"] + else: + return self.getSiteData(address, create)["auth_address"] + + def getAuthPrivatekey(self, address, create=True): + cert = self.getCert(address) + if cert: + return cert["auth_privatekey"] + else: + return self.getSiteData(address, create)["auth_privatekey"] + + # Add cert for the user + def addCert(self, auth_address, domain, auth_type, auth_user_name, cert_sign): + # Find privatekey by auth address + auth_privatekey = [site["auth_privatekey"] for site in list(self.sites.values()) if site["auth_address"] == auth_address][0] + cert_node = { + "auth_address": auth_address, + "auth_privatekey": auth_privatekey, + "auth_type": auth_type, + "auth_user_name": auth_user_name, + "cert_sign": cert_sign + } + # Check if we have already cert for that domain and its not the same + if self.certs.get(domain) and self.certs[domain] != cert_node: + return False + elif self.certs.get(domain) == cert_node: # Same, not updated + return None + else: # Not exist yet, add + self.certs[domain] = cert_node + self.save() + return True + + # Remove cert from user + def deleteCert(self, domain): + del self.certs[domain] + + # Set active cert for a site + def setCert(self, address, domain): + site_data = self.getSiteData(address) + if domain: + site_data["cert"] = domain + else: + if "cert" in site_data: + del site_data["cert"] + self.saveDelayed() + return site_data + + # Get cert for the site address + # Return: { "auth_address":.., "auth_privatekey":.., "auth_type": "web", "auth_user_name": "nofish", "cert_sign":.. } or None + def getCert(self, address): + site_data = self.getSiteData(address, create=False) + if not site_data or "cert" not in site_data: + return None # Site dont have cert + return self.certs.get(site_data["cert"]) + + # Get cert user name for the site address + # Return: user@certprovider.bit or None + def getCertUserId(self, address): + site_data = self.getSiteData(address, create=False) + if not site_data or "cert" not in site_data: + return None # Site dont have cert + cert = self.certs.get(site_data["cert"]) + if cert: + return cert["auth_user_name"] + "@" + site_data["cert"] diff --git a/src/User/UserManager.py b/src/User/UserManager.py new file mode 100644 index 00000000..067734a6 --- /dev/null +++ b/src/User/UserManager.py @@ -0,0 +1,77 @@ +# Included modules +import json +import logging +import time + +# ZeroNet Modules +from .User import User +from Plugin import PluginManager +from Config import config + + +@PluginManager.acceptPlugins +class UserManager(object): + def __init__(self): + self.users = {} + self.log = logging.getLogger("UserManager") + + # Load all user from data/users.json + def load(self): + if not self.users: + self.users = {} + + user_found = [] + added = 0 + s = time.time() + # Load new users + try: + json_path = "%s/users.json" % config.data_dir + data = json.load(open(json_path)) + except Exception as err: + raise Exception("Unable to load %s: %s" % (json_path, err)) + + for master_address, data in list(data.items()): + if master_address not in self.users: + user = User(master_address, data=data) + self.users[master_address] = user + added += 1 + user_found.append(master_address) + + # Remove deleted adresses + for master_address in list(self.users.keys()): + if master_address not in user_found: + del(self.users[master_address]) + self.log.debug("Removed user: %s" % master_address) + + if added: + self.log.debug("Added %s users in %.3fs" % (added, time.time() - s)) + + # Create new user + # Return: User + def create(self, master_address=None, master_seed=None): + self.list() # Load the users if it's not loaded yet + user = User(master_address, master_seed) + self.log.debug("Created user: %s" % user.master_address) + if user.master_address: # If successfully created + self.users[user.master_address] = user + user.saveDelayed() + return user + + # List all users from data/users.json + # Return: {"usermasteraddr": User} + def list(self): + if self.users == {}: # Not loaded yet + self.load() + return self.users + + # Get user based on master_address + # Return: User or None + def get(self, master_address=None): + users = self.list() + if users: + return list(users.values())[0] # Single user mode, always return the first + else: + return None + + +user_manager = UserManager() # Singleton diff --git a/src/User/__init__.py b/src/User/__init__.py new file mode 100644 index 00000000..4db9149e --- /dev/null +++ b/src/User/__init__.py @@ -0,0 +1 @@ +from .User import User diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py new file mode 100644 index 00000000..b7111ba1 --- /dev/null +++ b/src/Worker/Worker.py @@ -0,0 +1,239 @@ +import time + +import gevent +import gevent.lock + +from Debug import Debug +from Config import config +from Content.ContentManager import VerifyError + + +class WorkerDownloadError(Exception): + pass + + +class WorkerIOError(Exception): + pass + + +class WorkerStop(Exception): + pass + + +class Worker(object): + + def __init__(self, manager, peer): + self.manager = manager + self.peer = peer + self.task = None + self.key = None + self.running = False + self.thread = None + self.num_downloaded = 0 + self.num_failed = 0 + + def __str__(self): + return "Worker %s %s" % (self.manager.site.address_short, self.key) + + def __repr__(self): + return "<%s>" % self.__str__() + + def waitForTask(self, task, timeout): # Wait for other workers to finish the task + for sleep_i in range(1, timeout * 10): + time.sleep(0.1) + if task["done"] or task["workers_num"] == 0: + if config.verbose: + self.manager.log.debug("%s: %s, picked task free after %ss sleep. (done: %s)" % ( + self.key, task["inner_path"], 0.1 * sleep_i, task["done"] + )) + break + + if sleep_i % 10 == 0: + workers = self.manager.findWorkers(task) + if not workers or not workers[0].peer.connection: + break + worker_idle = time.time() - workers[0].peer.connection.last_recv_time + if worker_idle > 1: + if config.verbose: + self.manager.log.debug("%s: %s, worker %s seems idle, picked up task after %ss sleep. (done: %s)" % ( + self.key, task["inner_path"], workers[0].key, 0.1 * sleep_i, task["done"] + )) + break + return True + + def pickTask(self): # Find and select a new task for the worker + task = self.manager.getTask(self.peer) + if not task: # No more task + time.sleep(0.1) # Wait a bit for new tasks + task = self.manager.getTask(self.peer) + if not task: # Still no task, stop it + stats = "downloaded files: %s, failed: %s" % (self.num_downloaded, self.num_failed) + self.manager.log.debug("%s: No task found, stopping (%s)" % (self.key, stats)) + return False + + if not task["time_started"]: + task["time_started"] = time.time() # Task started now + + if task["workers_num"] > 0: # Wait a bit if someone already working on it + if task["peers"]: # It's an update + timeout = 3 + else: + timeout = 1 + + if task["size"] > 100 * 1024 * 1024: + timeout = timeout * 2 + + if config.verbose: + self.manager.log.debug("%s: Someone already working on %s (pri: %s), sleeping %s sec..." % ( + self.key, task["inner_path"], task["priority"], timeout + )) + + self.waitForTask(task, timeout) + return task + + def downloadTask(self, task): + try: + buff = self.peer.getFile(task["site"].address, task["inner_path"], task["size"]) + except Exception as err: + self.manager.log.debug("%s: getFile error: %s" % (self.key, err)) + raise WorkerDownloadError(str(err)) + + if not buff: + raise WorkerDownloadError("No response") + + return buff + + def getTaskLock(self, task): + if task["lock"] is None: + task["lock"] = gevent.lock.Semaphore() + return task["lock"] + + def writeTask(self, task, buff): + buff.seek(0) + try: + task["site"].storage.write(task["inner_path"], buff) + except Exception as err: + if type(err) == Debug.Notify: + self.manager.log.debug("%s: Write aborted: %s (%s: %s)" % (self.key, task["inner_path"], type(err), err)) + else: + self.manager.log.error("%s: Error writing: %s (%s: %s)" % (self.key, task["inner_path"], type(err), err)) + raise WorkerIOError(str(err)) + + def onTaskVerifyFail(self, task, error_message): + self.num_failed += 1 + if self.manager.started_task_num < 50 or config.verbose: + self.manager.log.debug( + "%s: Verify failed: %s, error: %s, failed peers: %s, workers: %s" % + (self.key, task["inner_path"], error_message, len(task["failed"]), task["workers_num"]) + ) + task["failed"].append(self.peer) + self.peer.hash_failed += 1 + if self.peer.hash_failed >= max(len(self.manager.tasks), 3) or self.peer.connection_error > 10: + # Broken peer: More fails than tasks number but atleast 3 + raise WorkerStop( + "Too many errors (hash failed: %s, connection error: %s)" % + (self.peer.hash_failed, self.peer.connection_error) + ) + + def handleTask(self, task): + download_err = write_err = False + + write_lock = None + try: + buff = self.downloadTask(task) + + if task["done"] is True: # Task done, try to find new one + return None + + if self.running is False: # Worker no longer needed or got killed + self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"])) + raise WorkerStop("Running got disabled") + + write_lock = self.getTaskLock(task) + write_lock.acquire() + if task["site"].content_manager.verifyFile(task["inner_path"], buff) is None: + is_same = True + else: + is_same = False + is_valid = True + except (WorkerDownloadError, VerifyError) as err: + download_err = err + is_valid = False + is_same = False + + if is_valid and not is_same: + if self.manager.started_task_num < 50 or task["priority"] > 10 or config.verbose: + self.manager.log.debug("%s: Verify correct: %s" % (self.key, task["inner_path"])) + try: + self.writeTask(task, buff) + except WorkerIOError as err: + write_err = err + + if not task["done"]: + if write_err: + self.manager.failTask(task, reason="Write error") + self.num_failed += 1 + self.manager.log.error("%s: Error writing %s: %s" % (self.key, task["inner_path"], write_err)) + elif is_valid: + self.manager.doneTask(task) + self.num_downloaded += 1 + + if write_lock is not None and write_lock.locked(): + write_lock.release() + + if not is_valid: + self.onTaskVerifyFail(task, download_err) + time.sleep(1) + return False + + return True + + def downloader(self): + self.peer.hash_failed = 0 # Reset hash error counter + while self.running: + # Try to pickup free file download task + task = self.pickTask() + + if not task: + break + + if task["done"]: + continue + + self.task = task + + self.manager.addTaskWorker(task, self) + + try: + success = self.handleTask(task) + except WorkerStop as err: + self.manager.log.debug("%s: Worker stopped: %s" % (self.key, err)) + self.manager.removeTaskWorker(task, self) + break + + self.manager.removeTaskWorker(task, self) + + self.peer.onWorkerDone() + self.running = False + self.manager.removeWorker(self) + + # Start the worker + def start(self): + self.running = True + self.thread = gevent.spawn(self.downloader) + + # Skip current task + def skip(self, reason="Unknown"): + self.manager.log.debug("%s: Force skipping (reason: %s)" % (self.key, reason)) + if self.thread: + self.thread.kill(exception=Debug.createNotifyType("Worker skipping (reason: %s)" % reason)) + self.start() + + # Force stop the worker + def stop(self, reason="Unknown"): + self.manager.log.debug("%s: Force stopping (reason: %s)" % (self.key, reason)) + self.running = False + if self.thread: + self.thread.kill(exception=Debug.createNotifyType("Worker stopped (reason: %s)" % reason)) + del self.thread + self.manager.removeWorker(self) diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py new file mode 100644 index 00000000..f68e8410 --- /dev/null +++ b/src/Worker/WorkerManager.py @@ -0,0 +1,600 @@ +import time +import logging +import collections + +import gevent + +from .Worker import Worker +from .WorkerTaskManager import WorkerTaskManager +from Config import config +from util import helper +from Plugin import PluginManager +from Debug.DebugLock import DebugLock +import util + + +@PluginManager.acceptPlugins +class WorkerManager(object): + + def __init__(self, site): + self.site = site + self.workers = {} # Key: ip:port, Value: Worker.Worker + self.tasks = WorkerTaskManager() + self.next_task_id = 1 + self.lock_add_task = DebugLock(name="Lock AddTask:%s" % self.site.address_short) + # {"id": 1, "evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "optional_hash_id": None, + # "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0, "failed": peer_ids, "lock": None or gevent.lock.RLock} + self.started_task_num = 0 # Last added task num + self.asked_peers = [] + self.running = True + self.time_task_added = 0 + self.log = logging.getLogger("WorkerManager:%s" % self.site.address_short) + self.site.greenlet_manager.spawn(self.checkTasks) + + def __str__(self): + return "WorkerManager %s" % self.site.address_short + + def __repr__(self): + return "<%s>" % self.__str__() + + # Check expired tasks + def checkTasks(self): + while self.running: + tasks = task = worker = workers = None # Cleanup local variables + announced = False + time.sleep(15) # Check every 15 sec + + # Clean up workers + for worker in list(self.workers.values()): + if worker.task and worker.task["done"]: + worker.skip(reason="Task done") # Stop workers with task done + + if not self.tasks: + continue + + tasks = self.tasks[:] # Copy it so removing elements wont cause any problem + num_tasks_started = len([task for task in tasks if task["time_started"]]) + + self.log.debug( + "Tasks: %s, started: %s, bad files: %s, total started: %s" % + (len(tasks), num_tasks_started, len(self.site.bad_files), self.started_task_num) + ) + + for task in tasks: + if task["time_started"] and time.time() >= task["time_started"] + 60: + self.log.debug("Timeout, Skipping: %s" % task) # Task taking too long time, skip it + # Skip to next file workers + workers = self.findWorkers(task) + if workers: + for worker in workers: + worker.skip(reason="Task timeout") + else: + self.failTask(task, reason="No workers") + + elif time.time() >= task["time_added"] + 60 and not self.workers: # No workers left + self.failTask(task, reason="Timeout") + + elif (task["time_started"] and time.time() >= task["time_started"] + 15) or not self.workers: + # Find more workers: Task started more than 15 sec ago or no workers + workers = self.findWorkers(task) + self.log.debug( + "Slow task: %s, (workers: %s, optional_hash_id: %s, peers: %s, failed: %s, asked: %s)" % + ( + task["inner_path"], len(workers), task["optional_hash_id"], + len(task["peers"] or []), len(task["failed"]), len(self.asked_peers) + ) + ) + if not announced and task["site"].isAddedRecently(): + task["site"].announce(mode="more") # Find more peers + announced = True + if task["optional_hash_id"]: + if self.workers: + if not task["time_started"]: + ask_limit = 20 + else: + ask_limit = max(10, time.time() - task["time_started"]) + if len(self.asked_peers) < ask_limit and len(task["peers"] or []) <= len(task["failed"]) * 2: + # Re-search for high priority + self.startFindOptional(find_more=True) + if task["peers"]: + peers_try = [peer for peer in task["peers"] if peer not in task["failed"] and peer not in workers] + if peers_try: + self.startWorkers(peers_try, force_num=5, reason="Task checker (optional, has peers)") + else: + self.startFindOptional(find_more=True) + else: + self.startFindOptional(find_more=True) + else: + if task["peers"]: # Release the peer lock + self.log.debug("Task peer lock release: %s" % task["inner_path"]) + task["peers"] = [] + self.startWorkers(reason="Task checker") + + if len(self.tasks) > len(self.workers) * 2 and len(self.workers) < self.getMaxWorkers(): + self.startWorkers(reason="Task checker (need more workers)") + + self.log.debug("checkTasks stopped running") + + # Returns the next free or less worked task + def getTask(self, peer): + for task in self.tasks: # Find a task + if task["peers"] and peer not in task["peers"]: + continue # This peer not allowed to pick this task + if peer in task["failed"]: + continue # Peer already tried to solve this, but failed + if task["optional_hash_id"] and task["peers"] is None: + continue # No peers found yet for the optional task + if task["done"]: + continue + return task + + def removeSolvedFileTasks(self, mark_as_good=True): + for task in self.tasks[:]: + if task["inner_path"] not in self.site.bad_files: + self.log.debug("No longer in bad_files, marking as %s: %s" % (mark_as_good, task["inner_path"])) + task["done"] = True + task["evt"].set(mark_as_good) + self.tasks.remove(task) + if not self.tasks: + self.started_task_num = 0 + self.site.updateWebsocket() + + # New peers added to site + def onPeers(self): + self.startWorkers(reason="More peers found") + + def getMaxWorkers(self): + if len(self.tasks) > 50: + return config.workers * 3 + else: + return config.workers + + # Add new worker + def addWorker(self, peer, multiplexing=False, force=False): + key = peer.key + if len(self.workers) > self.getMaxWorkers() and not force: + return False + if multiplexing: # Add even if we already have worker for this peer + key = "%s/%s" % (key, len(self.workers)) + if key not in self.workers: + # We dont have worker for that peer and workers num less than max + task = self.getTask(peer) + if task: + worker = Worker(self, peer) + self.workers[key] = worker + worker.key = key + worker.start() + return worker + else: + return False + else: # We have worker for this peer or its over the limit + return False + + def taskAddPeer(self, task, peer): + if task["peers"] is None: + task["peers"] = [] + if peer in task["failed"]: + return False + + if peer not in task["peers"]: + task["peers"].append(peer) + return True + + # Start workers to process tasks + def startWorkers(self, peers=None, force_num=0, reason="Unknown"): + if not self.tasks: + return False # No task for workers + max_workers = min(self.getMaxWorkers(), len(self.site.peers)) + if len(self.workers) >= max_workers and not peers: + return False # Workers number already maxed and no starting peers defined + self.log.debug( + "Starting workers (%s), tasks: %s, peers: %s, workers: %s" % + (reason, len(self.tasks), len(peers or []), len(self.workers)) + ) + if not peers: + peers = self.site.getConnectedPeers() + if len(peers) < max_workers: + peers += self.site.getRecentPeers(max_workers * 2) + if type(peers) is set: + peers = list(peers) + + # Sort by ping + peers.sort(key=lambda peer: peer.connection.last_ping_delay if peer.connection and peer.connection.last_ping_delay and len(peer.connection.waiting_requests) == 0 and peer.connection.connected else 9999) + + for peer in peers: # One worker for every peer + if peers and peer not in peers: + continue # If peers defined and peer not valid + + if force_num: + worker = self.addWorker(peer, force=True) + force_num -= 1 + else: + worker = self.addWorker(peer) + + if worker: + self.log.debug("Added worker: %s (rep: %s), workers: %s/%s" % (peer.key, peer.reputation, len(self.workers), max_workers)) + + # Find peers for optional hash in local hash tables and add to task peers + def findOptionalTasks(self, optional_tasks, reset_task=False): + found = collections.defaultdict(list) # { found_hash: [peer1, peer2...], ...} + + for peer in list(self.site.peers.values()): + if not peer.has_hashfield: + continue + + hashfield_set = set(peer.hashfield) # Finding in set is much faster + for task in optional_tasks: + optional_hash_id = task["optional_hash_id"] + if optional_hash_id in hashfield_set: + if reset_task and len(task["failed"]) > 0: + task["failed"] = [] + if peer in task["failed"]: + continue + if self.taskAddPeer(task, peer): + found[optional_hash_id].append(peer) + + return found + + # Find peers for optional hash ids in local hash tables + def findOptionalHashIds(self, optional_hash_ids, limit=0): + found = collections.defaultdict(list) # { found_hash_id: [peer1, peer2...], ...} + + for peer in list(self.site.peers.values()): + if not peer.has_hashfield: + continue + + hashfield_set = set(peer.hashfield) # Finding in set is much faster + for optional_hash_id in optional_hash_ids: + if optional_hash_id in hashfield_set: + found[optional_hash_id].append(peer) + if limit and len(found[optional_hash_id]) >= limit: + optional_hash_ids.remove(optional_hash_id) + + return found + + # Add peers to tasks from found result + def addOptionalPeers(self, found_ips): + found = collections.defaultdict(list) + for hash_id, peer_ips in found_ips.items(): + task = [task for task in self.tasks if task["optional_hash_id"] == hash_id] + if task: # Found task, lets take the first + task = task[0] + else: + continue + for peer_ip in peer_ips: + peer = self.site.addPeer(peer_ip[0], peer_ip[1], return_peer=True, source="optional") + if not peer: + continue + if self.taskAddPeer(task, peer): + found[hash_id].append(peer) + if peer.hashfield.appendHashId(hash_id): # Peer has this file + peer.time_hashfield = None # Peer hashfield probably outdated + + return found + + # Start find peers for optional files + @util.Noparallel(blocking=False, ignore_args=True) + def startFindOptional(self, reset_task=False, find_more=False, high_priority=False): + # Wait for more file requests + if len(self.tasks) < 20 or high_priority: + time.sleep(0.01) + elif len(self.tasks) > 90: + time.sleep(5) + else: + time.sleep(0.5) + + optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] + if not optional_tasks: + return False + optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) + time_tasks = self.time_task_added + + self.log.debug( + "Finding peers for optional files: %s (reset_task: %s, find_more: %s)" % + (optional_hash_ids, reset_task, find_more) + ) + found = self.findOptionalTasks(optional_tasks, reset_task=reset_task) + + if found: + found_peers = set([peer for peers in list(found.values()) for peer in peers]) + self.startWorkers(found_peers, force_num=3, reason="Optional found in local peers") + + if len(found) < len(optional_hash_ids) or find_more or (high_priority and any(len(peers) < 10 for peers in found.values())): + self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found))) + + # Query hashfield from connected peers + threads = [] + peers = self.site.getConnectedPeers() + if not peers: + peers = self.site.getConnectablePeers() + for peer in peers: + threads.append(self.site.greenlet_manager.spawn(peer.updateHashfield, force=find_more)) + gevent.joinall(threads, timeout=5) + + if time_tasks != self.time_task_added: # New task added since start + optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] + optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) + + found = self.findOptionalTasks(optional_tasks) + self.log.debug("Found optional files after query hashtable connected peers: %s/%s" % ( + len(found), len(optional_hash_ids) + )) + + if found: + found_peers = set([peer for hash_id_peers in list(found.values()) for peer in hash_id_peers]) + self.startWorkers(found_peers, force_num=3, reason="Optional found in connected peers") + + if len(found) < len(optional_hash_ids) or find_more: + self.log.debug( + "No connected hashtable result for optional files: %s (asked: %s)" % + (optional_hash_ids - set(found), len(self.asked_peers)) + ) + if not self.tasks: + self.log.debug("No tasks, stopping finding optional peers") + return + + # Try to query connected peers + threads = [] + peers = [peer for peer in self.site.getConnectedPeers() if peer.key not in self.asked_peers][0:10] + if not peers: + peers = self.site.getConnectablePeers(ignore=self.asked_peers) + + for peer in peers: + threads.append(self.site.greenlet_manager.spawn(peer.findHashIds, list(optional_hash_ids))) + self.asked_peers.append(peer.key) + + for i in range(5): + time.sleep(1) + + thread_values = [thread.value for thread in threads if thread.value] + if not thread_values: + continue + + found_ips = helper.mergeDicts(thread_values) + found = self.addOptionalPeers(found_ips) + self.log.debug("Found optional files after findhash connected peers: %s/%s (asked: %s)" % ( + len(found), len(optional_hash_ids), len(threads) + )) + + if found: + found_peers = set([peer for hash_id_peers in list(found.values()) for peer in hash_id_peers]) + self.startWorkers(found_peers, force_num=3, reason="Optional found by findhash connected peers") + + if len(thread_values) == len(threads): + # Got result from all started thread + break + + if len(found) < len(optional_hash_ids): + self.log.debug( + "No findHash result, try random peers: %s (asked: %s)" % + (optional_hash_ids - set(found), len(self.asked_peers)) + ) + # Try to query random peers + + if time_tasks != self.time_task_added: # New task added since start + optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] + optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) + + threads = [] + peers = self.site.getConnectablePeers(ignore=self.asked_peers) + + for peer in peers: + threads.append(self.site.greenlet_manager.spawn(peer.findHashIds, list(optional_hash_ids))) + self.asked_peers.append(peer.key) + + gevent.joinall(threads, timeout=15) + + found_ips = helper.mergeDicts([thread.value for thread in threads if thread.value]) + found = self.addOptionalPeers(found_ips) + self.log.debug("Found optional files after findhash random peers: %s/%s" % (len(found), len(optional_hash_ids))) + + if found: + found_peers = set([peer for hash_id_peers in list(found.values()) for peer in hash_id_peers]) + self.startWorkers(found_peers, force_num=3, reason="Option found using findhash random peers") + + if len(found) < len(optional_hash_ids): + self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found))) + + if time_tasks != self.time_task_added: # New task added since start + self.log.debug("New task since start, restarting...") + self.site.greenlet_manager.spawnLater(0.1, self.startFindOptional) + else: + self.log.debug("startFindOptional ended") + + # Stop all worker + def stopWorkers(self): + num = 0 + for worker in list(self.workers.values()): + worker.stop(reason="Stopping all workers") + num += 1 + tasks = self.tasks[:] # Copy + for task in tasks: # Mark all current task as failed + self.failTask(task, reason="Stopping all workers") + return num + + # Find workers by task + def findWorkers(self, task): + workers = [] + for worker in list(self.workers.values()): + if worker.task == task: + workers.append(worker) + return workers + + # Ends and remove a worker + def removeWorker(self, worker): + worker.running = False + if worker.key in self.workers: + del(self.workers[worker.key]) + self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), self.getMaxWorkers())) + if len(self.workers) <= self.getMaxWorkers() / 3 and len(self.asked_peers) < 10: + optional_task = next((task for task in self.tasks if task["optional_hash_id"]), None) + if optional_task: + if len(self.workers) == 0: + self.startFindOptional(find_more=True) + else: + self.startFindOptional() + elif self.tasks and not self.workers and worker.task and len(worker.task["failed"]) < 20: + self.log.debug("Starting new workers... (tasks: %s)" % len(self.tasks)) + self.startWorkers(reason="Removed worker") + + # Tasks sorted by this + def getPriorityBoost(self, inner_path): + if inner_path == "content.json": + return 9999 # Content.json always priority + if inner_path == "index.html": + return 9998 # index.html also important + if "-default" in inner_path: + return -4 # Default files are cloning not important + elif inner_path.endswith("all.css"): + return 14 # boost css files priority + elif inner_path.endswith("all.js"): + return 13 # boost js files priority + elif inner_path.endswith("dbschema.json"): + return 12 # boost database specification + elif inner_path.endswith("content.json"): + return 1 # boost included content.json files priority a bit + elif inner_path.endswith(".json"): + if len(inner_path) < 50: # Boost non-user json files + return 11 + else: + return 2 + return 0 + + def addTaskUpdate(self, task, peer, priority=0): + if priority > task["priority"]: + self.tasks.updateItem(task, "priority", priority) + if peer and task["peers"]: # This peer also has new version, add it to task possible peers + task["peers"].append(peer) + self.log.debug("Added peer %s to %s" % (peer.key, task["inner_path"])) + self.startWorkers([peer], reason="Added new task (update received by peer)") + elif peer and peer in task["failed"]: + task["failed"].remove(peer) # New update arrived, remove the peer from failed peers + self.log.debug("Removed peer %s from failed %s" % (peer.key, task["inner_path"])) + self.startWorkers([peer], reason="Added new task (peer failed before)") + + def addTaskCreate(self, inner_path, peer, priority=0, file_info=None): + evt = gevent.event.AsyncResult() + if peer: + peers = [peer] # Only download from this peer + else: + peers = None + if not file_info: + file_info = self.site.content_manager.getFileInfo(inner_path) + if file_info and file_info["optional"]: + optional_hash_id = helper.toHashId(file_info["sha512"]) + else: + optional_hash_id = None + if file_info: + size = file_info.get("size", 0) + else: + size = 0 + + self.lock_add_task.acquire() + + # Check again if we have task for this file + task = self.tasks.findTask(inner_path) + if task: + self.addTaskUpdate(task, peer, priority) + return task + + priority += self.getPriorityBoost(inner_path) + + if self.started_task_num == 0: # Boost priority for first requested file + priority += 1 + + task = { + "id": self.next_task_id, "evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, + "optional_hash_id": optional_hash_id, "time_added": time.time(), "time_started": None, "lock": None, + "time_action": None, "peers": peers, "priority": priority, "failed": [], "size": size + } + + self.tasks.append(task) + self.lock_add_task.release() + + self.next_task_id += 1 + self.started_task_num += 1 + if config.verbose: + self.log.debug( + "New task: %s, peer lock: %s, priority: %s, optional_hash_id: %s, tasks started: %s" % + (task["inner_path"], peers, priority, optional_hash_id, self.started_task_num) + ) + + self.time_task_added = time.time() + + if optional_hash_id: + if self.asked_peers: + del self.asked_peers[:] # Reset asked peers + self.startFindOptional(high_priority=priority > 0) + + if peers: + self.startWorkers(peers, reason="Added new optional task") + + else: + self.startWorkers(peers, reason="Added new task") + return task + + # Create new task and return asyncresult + def addTask(self, inner_path, peer=None, priority=0, file_info=None): + self.site.onFileStart(inner_path) # First task, trigger site download started + task = self.tasks.findTask(inner_path) + if task: # Already has task for that file + self.addTaskUpdate(task, peer, priority) + else: # No task for that file yet + task = self.addTaskCreate(inner_path, peer, priority, file_info) + return task + + def addTaskWorker(self, task, worker): + try: + self.tasks.updateItem(task, "workers_num", task["workers_num"] + 1) + except ValueError: + task["workers_num"] += 1 + + def removeTaskWorker(self, task, worker): + try: + self.tasks.updateItem(task, "workers_num", task["workers_num"] - 1) + except ValueError: + task["workers_num"] -= 1 + if len(task["failed"]) >= len(self.workers): + fail_reason = "Too many fails: %s (workers: %s)" % (len(task["failed"]), len(self.workers)) + self.failTask(task, reason=fail_reason) + + # Wait for other tasks + def checkComplete(self): + time.sleep(0.1) + if not self.tasks: + self.log.debug("Check complete: No tasks") + self.onComplete() + + def onComplete(self): + self.started_task_num = 0 + del self.asked_peers[:] + self.site.onComplete() # No more task trigger site complete + + # Mark a task done + def doneTask(self, task): + task["done"] = True + self.tasks.remove(task) # Remove from queue + if task["optional_hash_id"]: + self.log.debug( + "Downloaded optional file in %.3fs, adding to hashfield: %s" % + (time.time() - task["time_started"], task["inner_path"]) + ) + self.site.content_manager.optionalDownloaded(task["inner_path"], task["optional_hash_id"], task["size"]) + self.site.onFileDone(task["inner_path"]) + task["evt"].set(True) + if not self.tasks: + self.site.greenlet_manager.spawn(self.checkComplete) + + # Mark a task failed + def failTask(self, task, reason="Unknown"): + try: + self.tasks.remove(task) # Remove from queue + except ValueError as err: + return False + + self.log.debug("Task %s failed (Reason: %s)" % (task["inner_path"], reason)) + task["done"] = True + self.site.onFileFail(task["inner_path"]) + task["evt"].set(False) + if not self.tasks: + self.site.greenlet_manager.spawn(self.checkComplete) diff --git a/src/Worker/WorkerTaskManager.py b/src/Worker/WorkerTaskManager.py new file mode 100644 index 00000000..9359701d --- /dev/null +++ b/src/Worker/WorkerTaskManager.py @@ -0,0 +1,122 @@ +import bisect +from collections.abc import MutableSequence + + +class CustomSortedList(MutableSequence): + def __init__(self): + super().__init__() + self.items = [] # (priority, added index, actual value) + self.logging = False + + def __repr__(self): + return "<{0} {1}>".format(self.__class__.__name__, self.items) + + def __len__(self): + return len(self.items) + + def __getitem__(self, index): + if type(index) is int: + return self.items[index][2] + else: + return [item[2] for item in self.items[index]] + + def __delitem__(self, index): + del self.items[index] + + def __setitem__(self, index, value): + self.items[index] = self.valueToItem(value) + + def __str__(self): + return str(self[:]) + + def insert(self, index, value): + self.append(value) + + def append(self, value): + bisect.insort(self.items, self.valueToItem(value)) + + def updateItem(self, value, update_key=None, update_value=None): + self.remove(value) + if update_key is not None: + value[update_key] = update_value + self.append(value) + + def sort(self, *args, **kwargs): + raise Exception("Sorted list can't be sorted") + + def valueToItem(self, value): + return (self.getPriority(value), self.getId(value), value) + + def getPriority(self, value): + return value + + def getId(self, value): + return id(value) + + def indexSlow(self, value): + for pos, item in enumerate(self.items): + if item[2] == value: + return pos + return None + + def index(self, value): + item = (self.getPriority(value), self.getId(value), value) + bisect_pos = bisect.bisect(self.items, item) - 1 + if bisect_pos >= 0 and self.items[bisect_pos][2] == value: + return bisect_pos + + # Item probably changed since added, switch to slow iteration + pos = self.indexSlow(value) + + if self.logging: + print("Slow index for %s in pos %s bisect: %s" % (item[2], pos, bisect_pos)) + + if pos is None: + raise ValueError("%r not in list" % value) + else: + return pos + + def __contains__(self, value): + try: + self.index(value) + return True + except ValueError: + return False + + +class WorkerTaskManager(CustomSortedList): + def __init__(self): + super().__init__() + self.inner_paths = {} + + def getPriority(self, value): + return 0 - (value["priority"] - value["workers_num"] * 10) + + def getId(self, value): + return value["id"] + + def __contains__(self, value): + return value["inner_path"] in self.inner_paths + + def __delitem__(self, index): + # Remove from inner path cache + del self.inner_paths[self.items[index][2]["inner_path"]] + super().__delitem__(index) + + # Fast task search by inner_path + + def append(self, task): + if task["inner_path"] in self.inner_paths: + raise ValueError("File %s already has a task" % task["inner_path"]) + super().append(task) + # Create inner path cache for faster lookup by filename + self.inner_paths[task["inner_path"]] = task + + def remove(self, task): + if task not in self: + raise ValueError("%r not in list" % task) + else: + super().remove(task) + + def findTask(self, inner_path): + return self.inner_paths.get(inner_path, None) diff --git a/src/Worker/__init__.py b/src/Worker/__init__.py new file mode 100644 index 00000000..f4d20a96 --- /dev/null +++ b/src/Worker/__init__.py @@ -0,0 +1,2 @@ +from .Worker import Worker +from .WorkerManager import WorkerManager diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/lib/Ed25519.py b/src/lib/Ed25519.py new file mode 100644 index 00000000..20bdc1a9 --- /dev/null +++ b/src/lib/Ed25519.py @@ -0,0 +1,340 @@ +## ZeroNet onion V3 support +## The following copied code is copied from stem.util.ed25519 official Tor Project python3 lib +## url : https://gitweb.torproject.org/stem.git/tree/stem/util/ed25519.py +## the ##modified tag means that the function has been modified respect to the one used by stem lib +## the ##custom tag means that the function has been added by me and it's not present on the stem ed25519.py file +## every comment i make begins with ## +## +# The following is copied from... +# +# https://github.com/pyca/ed25519 +# +# This is under the CC0 license. For more information please see... +# +# https://github.com/pyca/cryptography/issues/5068 + +# ed25519.py - Optimized version of the reference implementation of Ed25519 +# +# Written in 2011? by Daniel J. Bernstein +# 2013 by Donald Stufft +# 2013 by Alex Gaynor +# 2013 by Greg Price +# +# To the extent possible under law, the author(s) have dedicated all copyright +# and related and neighboring rights to this software to the public domain +# worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication along +# with this software. If not, see +# . + +""" +NB: This code is not safe for use with secret keys or secret data. +The only safe use of this code is for verifying signatures on public messages. + +Functions for computing the public key of a secret key and for signing +a message are included, namely publickey_unsafe and signature_unsafe, +for testing purposes only. + +The root of the problem is that Python's long-integer arithmetic is +not designed for use in cryptography. Specifically, it may take more +or less time to execute an operation depending on the values of the +inputs, and its memory access patterns may also depend on the inputs. +This opens it to timing and cache side-channel attacks which can +disclose data to an attacker. We rely on Python's long-integer +arithmetic, so we cannot handle secrets without risking their disclosure. +""" + +import hashlib +import operator +import sys +import base64 + + +__version__ = "1.0.dev0" + + +# Useful for very coarse version differentiation. +PY3 = sys.version_info[0] == 3 + +if PY3: + indexbytes = operator.getitem + intlist2bytes = bytes + int2byte = operator.methodcaller("to_bytes", 1, "big") +else: + int2byte = chr + range = list(range(1,10000000)) + + def indexbytes(buf, i): + return ord(buf[i]) + + def intlist2bytes(l): + return b"".join(chr(c) for c in l) + + +b = 256 +q = 2 ** 255 - 19 +l = 2 ** 252 + 27742317777372353535851937790883648493 + + +def H(m): + return hashlib.sha512(m).digest() + + +def pow2(x, p): + """== pow(x, 2**p, q)""" + while p > 0: + x = x * x % q + p -= 1 + return x + + +def inv(z): + """$= z^{-1} \mod q$, for z != 0""" + # Adapted from curve25519_athlon.c in djb's Curve25519. + z2 = z * z % q # 2 + z9 = pow2(z2, 2) * z % q # 9 + z11 = z9 * z2 % q # 11 + z2_5_0 = (z11 * z11) % q * z9 % q # 31 == 2^5 - 2^0 + z2_10_0 = pow2(z2_5_0, 5) * z2_5_0 % q # 2^10 - 2^0 + z2_20_0 = pow2(z2_10_0, 10) * z2_10_0 % q # ... + z2_40_0 = pow2(z2_20_0, 20) * z2_20_0 % q + z2_50_0 = pow2(z2_40_0, 10) * z2_10_0 % q + z2_100_0 = pow2(z2_50_0, 50) * z2_50_0 % q + z2_200_0 = pow2(z2_100_0, 100) * z2_100_0 % q + z2_250_0 = pow2(z2_200_0, 50) * z2_50_0 % q # 2^250 - 2^0 + return pow2(z2_250_0, 5) * z11 % q # 2^255 - 2^5 + 11 = q - 2 + + +d = -121665 * inv(121666) % q +I = pow(2, (q - 1) // 4, q) + + +def xrecover(y): + xx = (y * y - 1) * inv(d * y * y + 1) + x = pow(xx, (q + 3) // 8, q) + + if (x * x - xx) % q != 0: + x = (x * I) % q + + if x % 2 != 0: + x = q-x + + return x + + +By = 4 * inv(5) +Bx = xrecover(By) +B = (Bx % q, By % q, 1, (Bx * By) % q) +ident = (0, 1, 1, 0) + + +def edwards_add(P, Q): + # This is formula sequence 'addition-add-2008-hwcd-3' from + # http://www.hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html + (x1, y1, z1, t1) = P + (x2, y2, z2, t2) = Q + + a = (y1-x1)*(y2-x2) % q + b = (y1+x1)*(y2+x2) % q + c = t1*2*d*t2 % q + dd = z1*2*z2 % q + e = b - a + f = dd - c + g = dd + c + h = b + a + x3 = e*f + y3 = g*h + t3 = e*h + z3 = f*g + + return (x3 % q, y3 % q, z3 % q, t3 % q) + + +def edwards_double(P): + # This is formula sequence 'dbl-2008-hwcd' from + # http://www.hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html + (x1, y1, z1, t1) = P + + a = x1*x1 % q + b = y1*y1 % q + c = 2*z1*z1 % q + # dd = -a + e = ((x1+y1)*(x1+y1) - a - b) % q + g = -a + b # dd + b + f = g - c + h = -a - b # dd - b + x3 = e*f + y3 = g*h + t3 = e*h + z3 = f*g + + return (x3 % q, y3 % q, z3 % q, t3 % q) + + +def scalarmult(P, e): + if e == 0: + return ident + Q = scalarmult(P, e // 2) + Q = edwards_double(Q) + if e & 1: + Q = edwards_add(Q, P) + return Q + + +# Bpow[i] == scalarmult(B, 2**i) +Bpow = [] + + +def make_Bpow(): + P = B + for i in range(253): + Bpow.append(P) + P = edwards_double(P) +make_Bpow() + + +def scalarmult_B(e): + """ + Implements scalarmult(B, e) more efficiently. + """ + # scalarmult(B, l) is the identity + e = e % l + P = ident + for i in range(253): + if e & 1: + P = edwards_add(P, Bpow[i]) + e = e // 2 + assert e == 0, e + return P + + +def encodeint(y): + bits = [(y >> i) & 1 for i in range(b)] + return b''.join([ + int2byte(sum([bits[i * 8 + j] << j for j in range(8)])) + for i in range(b//8) + ]) + + +def encodepoint(P): + (x, y, z, t) = P + zi = inv(z) + x = (x * zi) % q + y = (y * zi) % q + bits = [(y >> i) & 1 for i in range(b - 1)] + [x & 1] + return b''.join([ + int2byte(sum([bits[i * 8 + j] << j for j in range(8)])) + for i in range(b // 8) + ]) + + +def bit(h, i): + return (indexbytes(h, i // 8) >> (i % 8)) & 1 + +##modified +def publickey_unsafe(sk): + """ + Not safe to use with secret keys or secret data. + + See module docstring. This function should be used for testing only. + """ + ##h = H(sk) + h = sk + a = 2 ** (b - 2) + sum(2 ** i * bit(h, i) for i in range(3, b - 2)) + A = scalarmult_B(a) + return encodepoint(A) + +##custom +## from stem.util.str_tools._to_unicode_impl +## from https://gitweb.torproject.org/stem.git/tree/stem/util/str_tools.py#n80 +def to_unicode_impl(msg): + if msg is not None and not isinstance(msg, str): + return msg.decode('utf-8', 'replace') + else: + return msg + +##custom +## rewritten stem.descriptor.hidden_service.address_from_identity_key +## from https://gitweb.torproject.org/stem.git/tree/stem/descriptor/hidden_service.py#n1088 +def publickey_to_onionaddress(key): + CHECKSUM_CONSTANT = b'.onion checksum' + ## version = stem.client.datatype.Size.CHAR.pack(3) + version = b'\x03' + checksum = hashlib.sha3_256(CHECKSUM_CONSTANT + key + version).digest()[:2] + onion_address = base64.b32encode(key + checksum + version) + return to_unicode_impl(onion_address + b'.onion').lower() + + +def Hint(m): + h = H(m) + return sum(2 ** i * bit(h, i) for i in range(2 * b)) + +##modified +def signature_unsafe(m, sk, pk): + """ + Not safe to use with secret keys or secret data. + + See module docstring. This function should be used for testing only. + """ + ##h = H(sk) + h = sk + a = 2 ** (b - 2) + sum(2 ** i * bit(h, i) for i in range(3, b - 2)) + r = Hint( + intlist2bytes([indexbytes(h, j) for j in range(b // 8, b // 4)]) + m + ) + R = scalarmult_B(r) + S = (r + Hint(encodepoint(R) + pk + m) * a) % l + return encodepoint(R) + encodeint(S) + + +def isoncurve(P): + (x, y, z, t) = P + return (z % q != 0 and + x*y % q == z*t % q and + (y*y - x*x - z*z - d*t*t) % q == 0) + + +def decodeint(s): + return sum(2 ** i * bit(s, i) for i in range(0, b)) + + +def decodepoint(s): + y = sum(2 ** i * bit(s, i) for i in range(0, b - 1)) + x = xrecover(y) + if x & 1 != bit(s, b-1): + x = q - x + P = (x, y, 1, (x*y) % q) + if not isoncurve(P): + raise ValueError("decoding point that is not on curve") + return P + + +class SignatureMismatch(Exception): + pass + + +def checkvalid(s, m, pk): + """ + Not safe to use when any argument is secret. + + See module docstring. This function should be used only for + verifying public signatures of public messages. + """ + if len(s) != b // 4: + raise ValueError("signature length is wrong") + + if len(pk) != b // 8: + raise ValueError("public-key length is wrong") + + R = decodepoint(s[:b // 8]) + A = decodepoint(pk) + S = decodeint(s[b // 8:b // 4]) + h = Hint(encodepoint(R) + pk + m) + + (x1, y1, z1, t1) = P = scalarmult_B(S) + (x2, y2, z2, t2) = Q = edwards_add(R, scalarmult(A, h)) + + if (not isoncurve(P) or not isoncurve(Q) or + (x1*z2 - x2*z1) % q != 0 or (y1*z2 - y2*z1) % q != 0): + raise SignatureMismatch("signature does not pass verification") diff --git a/src/lib/__init__.py b/src/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/lib/bencode_open/LICENSE b/src/lib/bencode_open/LICENSE new file mode 100644 index 00000000..f0e46d71 --- /dev/null +++ b/src/lib/bencode_open/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Ivan Machugovskiy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/lib/bencode_open/__init__.py b/src/lib/bencode_open/__init__.py new file mode 100644 index 00000000..e3c783cc --- /dev/null +++ b/src/lib/bencode_open/__init__.py @@ -0,0 +1,160 @@ +def loads(data): + if not isinstance(data, bytes): + raise TypeError("Expected 'bytes' object, got {}".format(type(data))) + + offset = 0 + + + def parseInteger(): + nonlocal offset + + offset += 1 + had_digit = False + abs_value = 0 + + sign = 1 + if data[offset] == ord("-"): + sign = -1 + offset += 1 + while offset < len(data): + if data[offset] == ord("e"): + # End of string + offset += 1 + if not had_digit: + raise ValueError("Integer without value") + break + if ord("0") <= data[offset] <= ord("9"): + abs_value = abs_value * 10 + int(chr(data[offset])) + had_digit = True + offset += 1 + else: + raise ValueError("Invalid integer") + else: + raise ValueError("Unexpected EOF, expected integer") + + if not had_digit: + raise ValueError("Empty integer") + + return sign * abs_value + + + def parseString(): + nonlocal offset + + length = int(chr(data[offset])) + offset += 1 + + while offset < len(data): + if data[offset] == ord(":"): + offset += 1 + break + if ord("0") <= data[offset] <= ord("9"): + length = length * 10 + int(chr(data[offset])) + offset += 1 + else: + raise ValueError("Invalid string length") + else: + raise ValueError("Unexpected EOF, expected string contents") + + if offset + length > len(data): + raise ValueError("Unexpected EOF, expected string contents") + offset += length + + return data[offset - length:offset] + + + def parseList(): + nonlocal offset + + offset += 1 + values = [] + + while offset < len(data): + if data[offset] == ord("e"): + # End of list + offset += 1 + return values + else: + values.append(parse()) + + raise ValueError("Unexpected EOF, expected list contents") + + + def parseDict(): + nonlocal offset + + offset += 1 + items = {} + + while offset < len(data): + if data[offset] == ord("e"): + # End of list + offset += 1 + return items + else: + key, value = parse(), parse() + if not isinstance(key, bytes): + raise ValueError("A dict key must be a byte string") + if key in items: + raise ValueError("Duplicate dict key: {}".format(key)) + items[key] = value + + raise ValueError("Unexpected EOF, expected dict contents") + + + def parse(): + nonlocal offset + + if data[offset] == ord("i"): + return parseInteger() + elif data[offset] == ord("l"): + return parseList() + elif data[offset] == ord("d"): + return parseDict() + elif ord("0") <= data[offset] <= ord("9"): + return parseString() + + raise ValueError("Unknown type specifier: '{}'".format(chr(data[offset]))) + + result = parse() + + if offset != len(data): + raise ValueError("Expected EOF, got {} bytes left".format(len(data) - offset)) + + return result + + +def dumps(data): + result = bytearray() + + + def convert(data): + nonlocal result + + if isinstance(data, str): + raise ValueError("bencode only supports bytes, not str. Use encode") + + if isinstance(data, bytes): + result += str(len(data)).encode() + b":" + data + elif isinstance(data, int): + result += b"i" + str(data).encode() + b"e" + elif isinstance(data, list): + result += b"l" + for val in data: + convert(val) + result += b"e" + elif isinstance(data, dict): + result += b"d" + for key in sorted(data.keys()): + if not isinstance(key, bytes): + raise ValueError("Dict key can only be bytes, not {}".format(type(key))) + convert(key) + convert(data[key]) + result += b"e" + else: + raise ValueError("bencode only supports bytes, int, list and dict") + + + convert(data) + + return bytes(result) diff --git a/src/lib/cssvendor/__init__.py b/src/lib/cssvendor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/lib/cssvendor/cssvendor.py b/src/lib/cssvendor/cssvendor.py new file mode 100644 index 00000000..b04d7cc3 --- /dev/null +++ b/src/lib/cssvendor/cssvendor.py @@ -0,0 +1,39 @@ +import re + + +def prefix(content): + content = re.sub( + b"@keyframes (.*? {.*?}\s*})", b"@keyframes \\1\n@-webkit-keyframes \\1\n@-moz-keyframes \\1\n", + content, flags=re.DOTALL + ) + content = re.sub( + b'([^-\*])(border-radius|box-shadow|appearance|transition|animation|box-sizing|' + + b'backface-visibility|transform|filter|perspective|animation-[a-z-]+): (.*?)([;}])', + b'\\1-webkit-\\2: \\3; -moz-\\2: \\3; -o-\\2: \\3; -ms-\\2: \\3; \\2: \\3 \\4', content + ) + content = re.sub( + b'(?<=[^a-zA-Z0-9-])([a-zA-Z0-9-]+): {0,1}(linear-gradient)\((.*?)(\)[;\n])', + b'\\1: -webkit-\\2(\\3);' + + b'\\1: -moz-\\2(\\3);' + + b'\\1: -o-\\2(\\3);' + + b'\\1: -ms-\\2(\\3);' + + b'\\1: \\2(\\3);', content + ) + return content + +if __name__ == "__main__": + print(prefix(b""" + .test { + border-radius: 5px; + background: linear-gradient(red, blue); + } + + + @keyframes flip { + 0% { transform: perspective(120px) rotateX(0deg) rotateY(0deg); } + 50% { transform: perspective(120px) rotateX(-180.1deg) rotateY(0deg) } + 100% { transform: perspective(120px) rotateX(-180deg) rotateY(-179.9deg); } + } + + + """).decode("utf8")) diff --git a/src/lib/gevent_ws/__init__.py b/src/lib/gevent_ws/__init__.py new file mode 100644 index 00000000..a157e94c --- /dev/null +++ b/src/lib/gevent_ws/__init__.py @@ -0,0 +1,279 @@ +from gevent.pywsgi import WSGIHandler, _InvalidClientInput +from gevent.queue import Queue +import gevent +import hashlib +import base64 +import struct +import socket +import time +import sys + + +SEND_PACKET_SIZE = 1300 +OPCODE_TEXT = 1 +OPCODE_BINARY = 2 +OPCODE_CLOSE = 8 +OPCODE_PING = 9 +OPCODE_PONG = 10 +STATUS_OK = 1000 +STATUS_PROTOCOL_ERROR = 1002 +STATUS_DATA_ERROR = 1007 +STATUS_POLICY_VIOLATION = 1008 +STATUS_TOO_LONG = 1009 + + +class WebSocket: + def __init__(self, socket): + self.socket = socket + self.closed = False + self.status = None + self._receive_error = None + self._queue = Queue() + self.max_length = 10 * 1024 * 1024 + gevent.spawn(self._listen) + + + def set_max_message_length(self, length): + self.max_length = length + + + def _listen(self): + try: + while True: + fin = False + message = bytearray() + is_first_message = True + start_opcode = None + while not fin: + payload, opcode, fin = self._get_frame(max_length=self.max_length - len(message)) + # Make sure continuation frames have correct information + if not is_first_message and opcode != 0: + self._error(STATUS_PROTOCOL_ERROR) + if is_first_message: + if opcode not in (OPCODE_TEXT, OPCODE_BINARY): + self._error(STATUS_PROTOCOL_ERROR) + # Save opcode + start_opcode = opcode + message += payload + is_first_message = False + message = bytes(message) + if start_opcode == OPCODE_TEXT: # UTF-8 text + try: + message = message.decode() + except UnicodeDecodeError: + self._error(STATUS_DATA_ERROR) + self._queue.put(message) + except Exception as e: + self.closed = True + self._receive_error = e + self._queue.put(None) # To make sure the error is read + + + def receive(self): + if not self._queue.empty(): + return self.receive_nowait() + if isinstance(self._receive_error, EOFError): + return None + if self._receive_error: + raise self._receive_error + self._queue.peek() + return self.receive_nowait() + + + def receive_nowait(self): + ret = self._queue.get_nowait() + if self._receive_error and not isinstance(self._receive_error, EOFError): + raise self._receive_error + return ret + + + def send(self, data): + if self.closed: + raise EOFError() + if isinstance(data, str): + self._send_frame(OPCODE_TEXT, data.encode()) + elif isinstance(data, bytes): + self._send_frame(OPCODE_BINARY, data) + else: + raise TypeError("Expected str or bytes, got " + repr(type(data))) + + + # Reads a frame from the socket. Pings, pongs and close packets are handled + # automatically + def _get_frame(self, max_length): + while True: + payload, opcode, fin = self._read_frame(max_length=max_length) + if opcode == OPCODE_PING: + self._send_frame(OPCODE_PONG, payload) + elif opcode == OPCODE_PONG: + pass + elif opcode == OPCODE_CLOSE: + if len(payload) >= 2: + self.status = struct.unpack("!H", payload[:2])[0] + was_closed = self.closed + self.closed = True + if not was_closed: + # Send a close frame in response + self.close(STATUS_OK) + raise EOFError() + else: + return payload, opcode, fin + + + # Low-level function, use _get_frame instead + def _read_frame(self, max_length): + header = self._recv_exactly(2) + + if not (header[1] & 0x80): + self._error(STATUS_POLICY_VIOLATION) + + opcode = header[0] & 0xf + fin = bool(header[0] & 0x80) + + payload_length = header[1] & 0x7f + if payload_length == 126: + payload_length = struct.unpack("!H", self._recv_exactly(2))[0] + elif payload_length == 127: + payload_length = struct.unpack("!Q", self._recv_exactly(8))[0] + + # Control frames are handled in a special way + if opcode in (OPCODE_PING, OPCODE_PONG): + max_length = 125 + + if payload_length > max_length: + self._error(STATUS_TOO_LONG) + + mask = self._recv_exactly(4) + payload = self._recv_exactly(payload_length) + payload = self._unmask(payload, mask) + + return payload, opcode, fin + + + def _recv_exactly(self, length): + buf = bytearray() + while len(buf) < length: + block = self.socket.recv(min(4096, length - len(buf))) + if block == b"": + raise EOFError() + buf += block + return bytes(buf) + + + def _unmask(self, payload, mask): + def gen(c): + return bytes([x ^ c for x in range(256)]) + + + payload = bytearray(payload) + payload[0::4] = payload[0::4].translate(gen(mask[0])) + payload[1::4] = payload[1::4].translate(gen(mask[1])) + payload[2::4] = payload[2::4].translate(gen(mask[2])) + payload[3::4] = payload[3::4].translate(gen(mask[3])) + return bytes(payload) + + + def _send_frame(self, opcode, data): + for i in range(0, len(data), SEND_PACKET_SIZE): + part = data[i:i + SEND_PACKET_SIZE] + fin = int(i == (len(data) - 1) // SEND_PACKET_SIZE * SEND_PACKET_SIZE) + header = bytes( + [ + (opcode if i == 0 else 0) | (fin << 7), + min(len(part), 126) + ] + ) + if len(part) >= 126: + header += struct.pack("!H", len(part)) + self.socket.sendall(header + part) + + + def _error(self, status): + self.close(status) + raise EOFError() + + + def close(self, status=STATUS_OK): + self.closed = True + try: + self._send_frame(OPCODE_CLOSE, struct.pack("!H", status)) + except (BrokenPipeError, ConnectionResetError): + pass + self.socket.close() + + +class WebSocketHandler(WSGIHandler): + def handle_one_response(self): + self.time_start = time.time() + self.status = None + self.headers_sent = False + + self.result = None + self.response_use_chunked = False + self.response_length = 0 + + + http_connection = [s.strip().lower() for s in self.environ.get("HTTP_CONNECTION", "").split(",")] + if "upgrade" not in http_connection or self.environ.get("HTTP_UPGRADE", "").lower() != "websocket": + # Not my problem + return super(WebSocketHandler, self).handle_one_response() + + if "HTTP_SEC_WEBSOCKET_KEY" not in self.environ: + self.start_response("400 Bad Request", []) + return + + # Generate Sec-Websocket-Accept header + accept = self.environ["HTTP_SEC_WEBSOCKET_KEY"].encode() + accept += b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + accept = base64.b64encode(hashlib.sha1(accept).digest()).decode() + + # Accept + self.start_response("101 Switching Protocols", [ + ("Upgrade", "websocket"), + ("Connection", "Upgrade"), + ("Sec-Websocket-Accept", accept) + ])(b"") + + self.environ["wsgi.websocket"] = WebSocket(self.socket) + + # Can't call super because it sets invalid flags like "status" + try: + try: + self.run_application() + finally: + try: + self.wsgi_input._discard() + except (socket.error, IOError): + pass + except _InvalidClientInput: + self._send_error_response_if_possible(400) + except socket.error as ex: + if ex.args[0] in self.ignored_socket_errors: + self.close_connection = True + else: + self.handle_error(*sys.exc_info()) + except: # pylint:disable=bare-except + self.handle_error(*sys.exc_info()) + finally: + self.time_finish = time.time() + self.log_request() + self.close_connection = True + + + def process_result(self): + if "wsgi.websocket" in self.environ: + if self.result is None: + return + # Flushing result is required for werkzeug compatibility + for elem in self.result: + pass + else: + super(WebSocketHandler, self).process_result() + + + @property + def version(self): + if not self.environ: + return None + + return self.environ.get('HTTP_SEC_WEBSOCKET_VERSION') diff --git a/src/lib/libsecp256k1message/__init__.py b/src/lib/libsecp256k1message/__init__.py new file mode 100644 index 00000000..753f384e --- /dev/null +++ b/src/lib/libsecp256k1message/__init__.py @@ -0,0 +1 @@ +from .libsecp256k1message import * \ No newline at end of file diff --git a/src/lib/libsecp256k1message/libsecp256k1message.py b/src/lib/libsecp256k1message/libsecp256k1message.py new file mode 100644 index 00000000..59768b88 --- /dev/null +++ b/src/lib/libsecp256k1message/libsecp256k1message.py @@ -0,0 +1,162 @@ +import hashlib +import base64 +from coincurve import PrivateKey, PublicKey +from base58 import b58encode_check, b58decode_check +from hmac import compare_digest +from util.Electrum import format as zero_format + +RECID_MIN = 0 +RECID_MAX = 3 +RECID_UNCOMPR = 27 +LEN_COMPACT_SIG = 65 + +class SignatureError(ValueError): + pass + +def bitcoin_address(): + """Generate a public address and a secret address.""" + publickey, secretkey = key_pair() + + public_address = compute_public_address(publickey) + secret_address = compute_secret_address(secretkey) + + return (public_address, secret_address) + +def key_pair(): + """Generate a public key and a secret key.""" + secretkey = PrivateKey() + publickey = PublicKey.from_secret(secretkey.secret) + return (publickey, secretkey) + +def compute_public_address(publickey, compressed=False): + """Convert a public key to a public Bitcoin address.""" + public_plain = b'\x00' + public_digest(publickey, compressed=compressed) + return b58encode_check(public_plain) + +def compute_secret_address(secretkey): + """Convert a secret key to a secret Bitcoin address.""" + secret_plain = b'\x80' + secretkey.secret + return b58encode_check(secret_plain) + +def public_digest(publickey, compressed=False): + """Convert a public key to ripemd160(sha256()) digest.""" + publickey_hex = publickey.format(compressed=compressed) + return hashlib.new('ripemd160', hashlib.sha256(publickey_hex).digest()).digest() + +def address_public_digest(address): + """Convert a public Bitcoin address to ripemd160(sha256()) digest.""" + public_plain = b58decode_check(address) + if not public_plain.startswith(b'\x00') or len(public_plain) != 21: + raise ValueError('Invalid public key digest') + return public_plain[1:] + +def _decode_bitcoin_secret(address): + secret_plain = b58decode_check(address) + if not secret_plain.startswith(b'\x80') or len(secret_plain) != 33: + raise ValueError('Invalid secret key. Uncompressed keys only.') + return secret_plain[1:] + +def recover_public_key(signature, message): + """Recover public key from signature and message. + Recovered public key guarantees a correct signature""" + return PublicKey.from_signature_and_message(signature, message) + +def decode_secret_key(address): + """Convert a secret Bitcoin address to a secret key.""" + return PrivateKey(_decode_bitcoin_secret(address)) + + +def coincurve_sig(electrum_signature): + # coincurve := r + s + recovery_id + # where (0 <= recovery_id <= 3) + # https://github.com/bitcoin-core/secp256k1/blob/0b7024185045a49a1a6a4c5615bf31c94f63d9c4/src/modules/recovery/main_impl.h#L35 + if len(electrum_signature) != LEN_COMPACT_SIG: + raise ValueError('Not a 65-byte compact signature.') + # Compute coincurve recid + recid = (electrum_signature[0] - 27) & 3 + if not (RECID_MIN <= recid <= RECID_MAX): + raise ValueError('Recovery ID %d is not supported.' % recid) + recid_byte = int.to_bytes(recid, length=1, byteorder='big') + return electrum_signature[1:] + recid_byte + + +def electrum_sig(coincurve_signature): + # electrum := recovery_id + r + s + # where (27 <= recovery_id <= 30) + # https://github.com/scintill/bitcoin-signature-tools/blob/ed3f5be5045af74a54c92d3648de98c329d9b4f7/key.cpp#L285 + if len(coincurve_signature) != LEN_COMPACT_SIG: + raise ValueError('Not a 65-byte compact signature.') + # Compute Electrum recid + recid = coincurve_signature[-1] + RECID_UNCOMPR + if not (RECID_UNCOMPR + RECID_MIN <= recid <= RECID_UNCOMPR + RECID_MAX): + raise ValueError('Recovery ID %d is not supported.' % recid) + recid_byte = int.to_bytes(recid, length=1, byteorder='big') + return recid_byte + coincurve_signature[0:-1] + +def sign_data(secretkey, byte_string): + """Sign [byte_string] with [secretkey]. + Return serialized signature compatible with Electrum (ZeroNet).""" + # encode the message + encoded = zero_format(byte_string) + # sign the message and get a coincurve signature + signature = secretkey.sign_recoverable(encoded) + # reserialize signature and return it + return electrum_sig(signature) + +def verify_data(key_digest, electrum_signature, byte_string): + """Verify if [electrum_signature] of [byte_string] is correctly signed and + is signed with the secret counterpart of [key_digest]. + Raise SignatureError if the signature is forged or otherwise problematic.""" + # reserialize signature + signature = coincurve_sig(electrum_signature) + # encode the message + encoded = zero_format(byte_string) + # recover full public key from signature + # "which guarantees a correct signature" + publickey = recover_public_key(signature, encoded) + + # verify that the message is correctly signed by the public key + # correct_sig = verify_sig(publickey, signature, encoded) + + # verify that the public key is what we expect + correct_key = verify_key(publickey, key_digest) + + if not correct_key: + raise SignatureError('Signature is forged!') + +def verify_sig(publickey, signature, byte_string): + return publickey.verify(signature, byte_string) + +def verify_key(publickey, key_digest): + return compare_digest(key_digest, public_digest(publickey)) + +def recover_address(data, sign): + sign_bytes = base64.b64decode(sign) + is_compressed = ((sign_bytes[0] - 27) & 4) != 0 + publickey = recover_public_key(coincurve_sig(sign_bytes), zero_format(data)) + return compute_public_address(publickey, compressed=is_compressed) + +__all__ = [ + 'SignatureError', + 'key_pair', 'compute_public_address', 'compute_secret_address', + 'public_digest', 'address_public_digest', 'recover_public_key', 'decode_secret_key', + 'sign_data', 'verify_data', "recover_address" +] + +if __name__ == "__main__": + import base64, time, multiprocessing + s = time.time() + privatekey = decode_secret_key(b"5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk") + threads = [] + for i in range(1000): + data = bytes("hello", "utf8") + address = recover_address(data, "HGbib2kv9gm9IJjDt1FXbXFczZi35u0rZR3iPUIt5GglDDCeIQ7v8eYXVNIaLoJRI4URGZrhwmsYQ9aVtRTnTfQ=") + print("- Verify x10000: %.3fs %s" % (time.time() - s, address)) + + s = time.time() + for i in range(1000): + privatekey = decode_secret_key(b"5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk") + sign = sign_data(privatekey, b"hello") + sign_b64 = base64.b64encode(sign) + + print("- Sign x1000: %.3fs" % (time.time() - s)) diff --git a/src/lib/openssl/openssl.cnf b/src/lib/openssl/openssl.cnf new file mode 100644 index 00000000..1c1ec47f --- /dev/null +++ b/src/lib/openssl/openssl.cnf @@ -0,0 +1,58 @@ +[ req ] +default_bits = 2048 +default_keyfile = server-key.pem +distinguished_name = subject +req_extensions = req_ext +x509_extensions = x509_ext +string_mask = utf8only + +# The Subject DN can be formed using X501 or RFC 4514 (see RFC 4519 for a description). +# Its sort of a mashup. For example, RFC 4514 does not provide emailAddress. +[ subject ] +countryName = US +stateOrProvinceName = NY +localityName = New York +organizationName = Example, LLC + +# Use a friendly name here because its presented to the user. The server's DNS +# names are placed in Subject Alternate Names. Plus, DNS names here is deprecated +# by both IETF and CA/Browser Forums. If you place a DNS name here, then you +# must include the DNS name in the SAN too (otherwise, Chrome and others that +# strictly follow the CA/Browser Baseline Requirements will fail). +commonName = Example Company + +emailAddress = test@example.com + +# Section x509_ext is used when generating a self-signed certificate. I.e., openssl req -x509 ... +[ x509_ext ] + +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer + +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alternate_names + +# RFC 5280, Section 4.2.1.12 makes EKU optional +# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused +# extendedKeyUsage = serverAuth, clientAuth + +# Section req_ext is used when generating a certificate signing request. I.e., openssl req ... +[ req_ext ] + +subjectKeyIdentifier = hash + +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alternate_names + +# RFC 5280, Section 4.2.1.12 makes EKU optional +# CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused +# extendedKeyUsage = serverAuth, clientAuth + +[ alternate_names ] + +DNS.1 = $ENV::CN +DNS.2 = www.$ENV::CN \ No newline at end of file diff --git a/src/lib/pyaes/LICENSE.txt b/src/lib/pyaes/LICENSE.txt new file mode 100644 index 00000000..0417a6c2 --- /dev/null +++ b/src/lib/pyaes/LICENSE.txt @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Richard Moore + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/src/lib/pyaes/README.md b/src/lib/pyaes/README.md new file mode 100644 index 00000000..26e3b2ba --- /dev/null +++ b/src/lib/pyaes/README.md @@ -0,0 +1,363 @@ +pyaes +===== + +A pure-Python implementation of the AES block cipher algorithm and the common modes of operation (CBC, CFB, CTR, ECB and OFB). + + +Features +-------- + +* Supports all AES key sizes +* Supports all AES common modes +* Pure-Python (no external dependencies) +* BlockFeeder API allows streams to easily be encrypted and decrypted +* Python 2.x and 3.x support (make sure you pass in bytes(), not strings for Python 3) + + +API +--- + +All keys may be 128 bits (16 bytes), 192 bits (24 bytes) or 256 bits (32 bytes) long. + +To generate a random key use: +```python +import os + +# 128 bit, 192 bit and 256 bit keys +key_128 = os.urandom(16) +key_192 = os.urandom(24) +key_256 = os.urandom(32) +``` + +To generate keys from simple-to-remember passwords, consider using a _password-based key-derivation function_ such as [scrypt](https://github.com/ricmoo/pyscrypt). + + +### Common Modes of Operation + +There are many modes of operations, each with various pros and cons. In general though, the **CBC** and **CTR** modes are recommended. The **ECB is NOT recommended.**, and is included primarily for completeness. + +Each of the following examples assumes the following key: +```python +import pyaes + +# A 256 bit (32 byte) key +key = "This_key_for_demo_purposes_only!" + +# For some modes of operation we need a random initialization vector +# of 16 bytes +iv = "InitializationVe" +``` + + +#### Counter Mode of Operation (recommended) + +```python +aes = pyaes.AESModeOfOperationCTR(key) +plaintext = "Text may be any length you wish, no padding is required" +ciphertext = aes.encrypt(plaintext) + +# '''\xb6\x99\x10=\xa4\x96\x88\xd1\x89\x1co\xe6\x1d\xef;\x11\x03\xe3\xee +# \xa9V?wY\xbfe\xcdO\xe3\xdf\x9dV\x19\xe5\x8dk\x9fh\xb87>\xdb\xa3\xd6 +# \x86\xf4\xbd\xb0\x97\xf1\t\x02\xe9 \xed''' +print repr(ciphertext) + +# The counter mode of operation maintains state, so decryption requires +# a new instance be created +aes = pyaes.AESModeOfOperationCTR(key) +decrypted = aes.decrypt(ciphertext) + +# True +print decrypted == plaintext + +# To use a custom initial value +counter = pyaes.Counter(initial_value = 100) +aes = pyaes.AESModeOfOperationCTR(key, counter = counter) +ciphertext = aes.encrypt(plaintext) + +# '''WZ\x844\x02\xbfoY\x1f\x12\xa6\xce\x03\x82Ei)\xf6\x97mX\x86\xe3\x9d +# _1\xdd\xbd\x87\xb5\xccEM_4\x01$\xa6\x81\x0b\xd5\x04\xd7Al\x07\xe5 +# \xb2\x0e\\\x0f\x00\x13,\x07''' +print repr(ciphertext) +``` + + +#### Cipher-Block Chaining (recommended) + +```python +aes = pyaes.AESModeOfOperationCBC(key, iv = iv) +plaintext = "TextMustBe16Byte" +ciphertext = aes.encrypt(plaintext) + +# '\xd6:\x18\xe6\xb1\xb3\xc3\xdc\x87\xdf\xa7|\x08{k\xb6' +print repr(ciphertext) + + +# The cipher-block chaining mode of operation maintains state, so +# decryption requires a new instance be created +aes = pyaes.AESModeOfOperationCBC(key, iv = iv) +decrypted = aes.decrypt(ciphertext) + +# True +print decrypted == plaintext +``` + + +#### Cipher Feedback + +```python +# Each block into the mode of operation must be a multiple of the segment +# size. For this example we choose 8 bytes. +aes = pyaes.AESModeOfOperationCFB(key, iv = iv, segment_size = 8) +plaintext = "TextMustBeAMultipleOfSegmentSize" +ciphertext = aes.encrypt(plaintext) + +# '''v\xa9\xc1w"\x8aL\x93\xcb\xdf\xa0/\xf8Y\x0b\x8d\x88i\xcb\x85rmp +# \x85\xfe\xafM\x0c)\xd5\xeb\xaf''' +print repr(ciphertext) + + +# The cipher-block chaining mode of operation maintains state, so +# decryption requires a new instance be created +aes = pyaes.AESModeOfOperationCFB(key, iv = iv, segment_size = 8) +decrypted = aes.decrypt(ciphertext) + +# True +print decrypted == plaintext +``` + + +#### Output Feedback Mode of Operation + +```python +aes = pyaes.AESModeOfOperationOFB(key, iv = iv) +plaintext = "Text may be any length you wish, no padding is required" +ciphertext = aes.encrypt(plaintext) + +# '''v\xa9\xc1wO\x92^\x9e\rR\x1e\xf7\xb1\xa2\x9d"l1\xc7\xe7\x9d\x87(\xc26s +# \xdd8\xc8@\xb6\xd9!\xf5\x0cM\xaa\x9b\xc4\xedLD\xe4\xb9\xd8\xdf\x9e\xac +# \xa1\xb8\xea\x0f\x8ev\xb5''' +print repr(ciphertext) + +# The counter mode of operation maintains state, so decryption requires +# a new instance be created +aes = pyaes.AESModeOfOperationOFB(key, iv = iv) +decrypted = aes.decrypt(ciphertext) + +# True +print decrypted == plaintext +``` + + +#### Electronic Codebook (NOT recommended) + +```python +aes = pyaes.AESModeOfOperationECB(key) +plaintext = "TextMustBe16Byte" +ciphertext = aes.encrypt(plaintext) + +# 'L6\x95\x85\xe4\xd9\xf1\x8a\xfb\xe5\x94X\x80|\x19\xc3' +print repr(ciphertext) + +# Since there is no state stored in this mode of operation, it +# is not necessary to create a new aes object for decryption. +#aes = pyaes.AESModeOfOperationECB(key) +decrypted = aes.decrypt(ciphertext) + +# True +print decrypted == plaintext +``` + + +### BlockFeeder + +Since most of the modes of operations require data in specific block-sized or segment-sized blocks, it can be difficult when working with large arbitrary streams or strings of data. + +The BlockFeeder class is meant to make life easier for you, by buffering bytes across multiple calls and returning bytes as they are available, as well as padding or stripping the output when finished, if necessary. + +```python +import pyaes + +# Any mode of operation can be used; for this example CBC +key = "This_key_for_demo_purposes_only!" +iv = "InitializationVe" + +ciphertext = '' + +# We can encrypt one line at a time, regardles of length +encrypter = pyaes.Encrypter(pyaes.AESModeOfOperationCBC(key, iv)) +for line in file('/etc/passwd'): + ciphertext += encrypter.feed(line) + +# Make a final call to flush any remaining bytes and add paddin +ciphertext += encrypter.feed() + +# We can decrypt the cipher text in chunks (here we split it in half) +decrypter = pyaes.Decrypter(pyaes.AESModeOfOperationCBC(key, iv)) +decrypted = decrypter.feed(ciphertext[:len(ciphertext) / 2]) +decrypted += decrypter.feed(ciphertext[len(ciphertext) / 2:]) + +# Again, make a final call to flush any remaining bytes and strip padding +decrypted += decrypter.feed() + +print file('/etc/passwd').read() == decrypted +``` + +### Stream Feeder + +This is meant to make it even easier to encrypt and decrypt streams and large files. + +```python +import pyaes + +# Any mode of operation can be used; for this example CTR +key = "This_key_for_demo_purposes_only!" + +# Create the mode of operation to encrypt with +mode = pyaes.AESModeOfOperationCTR(key) + +# The input and output files +file_in = file('/etc/passwd') +file_out = file('/tmp/encrypted.bin', 'wb') + +# Encrypt the data as a stream, the file is read in 8kb chunks, be default +pyaes.encrypt_stream(mode, file_in, file_out) + +# Close the files +file_in.close() +file_out.close() +``` + +Decrypting is identical, except you would use `pyaes.decrypt_stream`, and the encrypted file would be the `file_in` and target for decryption the `file_out`. + +### AES block cipher + +Generally you should use one of the modes of operation above. This may however be useful for experimenting with a custom mode of operation or dealing with encrypted blocks. + +The block cipher requires exactly one block of data to encrypt or decrypt, and each block should be an array with each element an integer representation of a byte. + +```python +import pyaes + +# 16 byte block of plain text +plaintext = "Hello World!!!!!" +plaintext_bytes = [ ord(c) for c in plaintext ] + +# 32 byte key (256 bit) +key = "This_key_for_demo_purposes_only!" + +# Our AES instance +aes = pyaes.AES(key) + +# Encrypt! +ciphertext = aes.encrypt(plaintext_bytes) + +# [55, 250, 182, 25, 185, 208, 186, 95, 206, 115, 50, 115, 108, 58, 174, 115] +print repr(ciphertext) + +# Decrypt! +decrypted = aes.decrypt(ciphertext) + +# True +print decrypted == plaintext_bytes +``` + +What is a key? +-------------- + +This seems to be a point of confusion for many people new to using encryption. You can think of the key as the *"password"*. However, these algorithms require the *"password"* to be a specific length. + +With AES, there are three possible key lengths, 16-bytes, 24-bytes or 32-bytes. When you create an AES object, the key size is automatically detected, so it is important to pass in a key of the correct length. + +Often, you wish to provide a password of arbitrary length, for example, something easy to remember or write down. In these cases, you must come up with a way to transform the password into a key, of a specific length. A **Password-Based Key Derivation Function** (PBKDF) is an algorithm designed for this exact purpose. + +Here is an example, using the popular (possibly obsolete?) *crypt* PBKDF: + +``` +# See: https://www.dlitz.net/software/python-pbkdf2/ +import pbkdf2 + +password = "HelloWorld" + +# The crypt PBKDF returns a 48-byte string +key = pbkdf2.crypt(password) + +# A 16-byte, 24-byte and 32-byte key, respectively +key_16 = key[:16] +key_24 = key[:24] +key_32 = key[:32] +``` + +The [scrypt](https://github.com/ricmoo/pyscrypt) PBKDF is intentionally slow, to make it more difficult to brute-force guess a password: + +``` +# See: https://github.com/ricmoo/pyscrypt +import pyscrypt + +password = "HelloWorld" + +# Salt is required, and prevents Rainbow Table attacks +salt = "SeaSalt" + +# N, r, and p are parameters to specify how difficult it should be to +# generate a key; bigger numbers take longer and more memory +N = 1024 +r = 1 +p = 1 + +# A 16-byte, 24-byte and 32-byte key, respectively; the scrypt algorithm takes +# a 6-th parameter, indicating key length +key_16 = pyscrypt.hash(password, salt, N, r, p, 16) +key_24 = pyscrypt.hash(password, salt, N, r, p, 24) +key_32 = pyscrypt.hash(password, salt, N, r, p, 32) +``` + +Another possibility, is to use a hashing function, such as SHA256 to hash the password, but this method may be vulnerable to [Rainbow Attacks](http://en.wikipedia.org/wiki/Rainbow_table), unless you use a [salt](http://en.wikipedia.org/wiki/Salt_(cryptography)). + +```python +import hashlib + +password = "HelloWorld" + +# The SHA256 hash algorithm returns a 32-byte string +hashed = hashlib.sha256(password).digest() + +# A 16-byte, 24-byte and 32-byte key, respectively +key_16 = hashed[:16] +key_24 = hashed[:24] +key_32 = hashed +``` + + + + +Performance +----------- + +There is a test case provided in _/tests/test-aes.py_ which does some basic performance testing (its primary purpose is moreso as a regression test). + +Based on that test, in **CPython**, this library is about 30x slower than [PyCrypto](https://www.dlitz.net/software/pycrypto/) for CBC, ECB and OFB; about 80x slower for CFB; and 300x slower for CTR. + +Based on that same test, in **Pypy**, this library is about 4x slower than [PyCrypto](https://www.dlitz.net/software/pycrypto/) for CBC, ECB and OFB; about 12x slower for CFB; and 19x slower for CTR. + +The PyCrypto documentation makes reference to the counter call being responsible for the speed problems of the counter (CTR) mode of operation, which is why they use a specially optimized counter. I will investigate this problem further in the future. + + +FAQ +--- + +#### Why do this? + +The short answer, *why not?* + +The longer answer, is for my [pyscrypt](https://github.com/ricmoo/pyscrypt) library. I required a pure-Python AES implementation that supported 256-bit keys with the counter (CTR) mode of operation. After searching, I found several implementations, but all were missing CTR or only supported 128 bit keys. After all the work of learning AES inside and out to implement the library, it was only a marginal amount of extra work to library-ify a more general solution. So, *why not?* + +#### How do I get a question I have added? + +E-mail me at pyaes@ricmoo.com with any questions, suggestions, comments, et cetera. + + +#### Can I give you my money? + +Umm... Ok? :-) + +_Bitcoin_ - `18UDs4qV1shu2CgTS2tKojhCtM69kpnWg9` diff --git a/src/lib/pyaes/__init__.py b/src/lib/pyaes/__init__.py new file mode 100644 index 00000000..5712f794 --- /dev/null +++ b/src/lib/pyaes/__init__.py @@ -0,0 +1,53 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014 Richard Moore +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# This is a pure-Python implementation of the AES algorithm and AES common +# modes of operation. + +# See: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +# See: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation + + +# Supported key sizes: +# 128-bit +# 192-bit +# 256-bit + + +# Supported modes of operation: +# ECB - Electronic Codebook +# CBC - Cipher-Block Chaining +# CFB - Cipher Feedback +# OFB - Output Feedback +# CTR - Counter + +# See the README.md for API details and general information. + +# Also useful, PyCrypto, a crypto library implemented in C with Python bindings: +# https://www.dlitz.net/software/pycrypto/ + + +VERSION = [1, 3, 0] + +from .aes import AES, AESModeOfOperationCTR, AESModeOfOperationCBC, AESModeOfOperationCFB, AESModeOfOperationECB, AESModeOfOperationOFB, AESModesOfOperation, Counter +from .blockfeeder import decrypt_stream, Decrypter, encrypt_stream, Encrypter +from .blockfeeder import PADDING_NONE, PADDING_DEFAULT diff --git a/src/lib/pyaes/aes.py b/src/lib/pyaes/aes.py new file mode 100644 index 00000000..c6e8bc02 --- /dev/null +++ b/src/lib/pyaes/aes.py @@ -0,0 +1,589 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014 Richard Moore +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# This is a pure-Python implementation of the AES algorithm and AES common +# modes of operation. + +# See: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard + +# Honestly, the best description of the modes of operations are the wonderful +# diagrams on Wikipedia. They explain in moments what my words could never +# achieve. Hence the inline documentation here is sparer than I'd prefer. +# See: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation + +# Also useful, PyCrypto, a crypto library implemented in C with Python bindings: +# https://www.dlitz.net/software/pycrypto/ + + +# Supported key sizes: +# 128-bit +# 192-bit +# 256-bit + + +# Supported modes of operation: +# ECB - Electronic Codebook +# CBC - Cipher-Block Chaining +# CFB - Cipher Feedback +# OFB - Output Feedback +# CTR - Counter + + +# See the README.md for API details and general information. + + +import copy +import struct + +__all__ = ["AES", "AESModeOfOperationCTR", "AESModeOfOperationCBC", "AESModeOfOperationCFB", + "AESModeOfOperationECB", "AESModeOfOperationOFB", "AESModesOfOperation", "Counter"] + + +def _compact_word(word): + return (word[0] << 24) | (word[1] << 16) | (word[2] << 8) | word[3] + +def _string_to_bytes(text): + return list(ord(c) for c in text) + +def _bytes_to_string(binary): + return "".join(chr(b) for b in binary) + +def _concat_list(a, b): + return a + b + + +# Python 3 compatibility +try: + xrange +except Exception: + xrange = range + + # Python 3 supports bytes, which is already an array of integers + def _string_to_bytes(text): + if isinstance(text, bytes): + return text + return [ord(c) for c in text] + + # In Python 3, we return bytes + def _bytes_to_string(binary): + return bytes(binary) + + # Python 3 cannot concatenate a list onto a bytes, so we bytes-ify it first + def _concat_list(a, b): + return a + bytes(b) + + +# Based *largely* on the Rijndael implementation +# See: http://csrc.nist.gov/publications/fips/fips197/fips-197.pdf +class AES(object): + '''Encapsulates the AES block cipher. + + You generally should not need this. Use the AESModeOfOperation classes + below instead.''' + + # Number of rounds by keysize + number_of_rounds = {16: 10, 24: 12, 32: 14} + + # Round constant words + rcon = [ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91 ] + + # S-box and Inverse S-box (S is for Substitution) + S = [ 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 ] + Si =[ 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d ] + + # Transformations for encryption + T1 = [ 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a ] + T2 = [ 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616 ] + T3 = [ 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16 ] + T4 = [ 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c ] + + # Transformations for decryption + T5 = [ 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742 ] + T6 = [ 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857 ] + T7 = [ 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8 ] + T8 = [ 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0 ] + + # Transformations for decryption key expansion + U1 = [ 0x00000000, 0x0e090d0b, 0x1c121a16, 0x121b171d, 0x3824342c, 0x362d3927, 0x24362e3a, 0x2a3f2331, 0x70486858, 0x7e416553, 0x6c5a724e, 0x62537f45, 0x486c5c74, 0x4665517f, 0x547e4662, 0x5a774b69, 0xe090d0b0, 0xee99ddbb, 0xfc82caa6, 0xf28bc7ad, 0xd8b4e49c, 0xd6bde997, 0xc4a6fe8a, 0xcaaff381, 0x90d8b8e8, 0x9ed1b5e3, 0x8ccaa2fe, 0x82c3aff5, 0xa8fc8cc4, 0xa6f581cf, 0xb4ee96d2, 0xbae79bd9, 0xdb3bbb7b, 0xd532b670, 0xc729a16d, 0xc920ac66, 0xe31f8f57, 0xed16825c, 0xff0d9541, 0xf104984a, 0xab73d323, 0xa57ade28, 0xb761c935, 0xb968c43e, 0x9357e70f, 0x9d5eea04, 0x8f45fd19, 0x814cf012, 0x3bab6bcb, 0x35a266c0, 0x27b971dd, 0x29b07cd6, 0x038f5fe7, 0x0d8652ec, 0x1f9d45f1, 0x119448fa, 0x4be30393, 0x45ea0e98, 0x57f11985, 0x59f8148e, 0x73c737bf, 0x7dce3ab4, 0x6fd52da9, 0x61dc20a2, 0xad766df6, 0xa37f60fd, 0xb16477e0, 0xbf6d7aeb, 0x955259da, 0x9b5b54d1, 0x894043cc, 0x87494ec7, 0xdd3e05ae, 0xd33708a5, 0xc12c1fb8, 0xcf2512b3, 0xe51a3182, 0xeb133c89, 0xf9082b94, 0xf701269f, 0x4de6bd46, 0x43efb04d, 0x51f4a750, 0x5ffdaa5b, 0x75c2896a, 0x7bcb8461, 0x69d0937c, 0x67d99e77, 0x3daed51e, 0x33a7d815, 0x21bccf08, 0x2fb5c203, 0x058ae132, 0x0b83ec39, 0x1998fb24, 0x1791f62f, 0x764dd68d, 0x7844db86, 0x6a5fcc9b, 0x6456c190, 0x4e69e2a1, 0x4060efaa, 0x527bf8b7, 0x5c72f5bc, 0x0605bed5, 0x080cb3de, 0x1a17a4c3, 0x141ea9c8, 0x3e218af9, 0x302887f2, 0x223390ef, 0x2c3a9de4, 0x96dd063d, 0x98d40b36, 0x8acf1c2b, 0x84c61120, 0xaef93211, 0xa0f03f1a, 0xb2eb2807, 0xbce2250c, 0xe6956e65, 0xe89c636e, 0xfa877473, 0xf48e7978, 0xdeb15a49, 0xd0b85742, 0xc2a3405f, 0xccaa4d54, 0x41ecdaf7, 0x4fe5d7fc, 0x5dfec0e1, 0x53f7cdea, 0x79c8eedb, 0x77c1e3d0, 0x65daf4cd, 0x6bd3f9c6, 0x31a4b2af, 0x3fadbfa4, 0x2db6a8b9, 0x23bfa5b2, 0x09808683, 0x07898b88, 0x15929c95, 0x1b9b919e, 0xa17c0a47, 0xaf75074c, 0xbd6e1051, 0xb3671d5a, 0x99583e6b, 0x97513360, 0x854a247d, 0x8b432976, 0xd134621f, 0xdf3d6f14, 0xcd267809, 0xc32f7502, 0xe9105633, 0xe7195b38, 0xf5024c25, 0xfb0b412e, 0x9ad7618c, 0x94de6c87, 0x86c57b9a, 0x88cc7691, 0xa2f355a0, 0xacfa58ab, 0xbee14fb6, 0xb0e842bd, 0xea9f09d4, 0xe49604df, 0xf68d13c2, 0xf8841ec9, 0xd2bb3df8, 0xdcb230f3, 0xcea927ee, 0xc0a02ae5, 0x7a47b13c, 0x744ebc37, 0x6655ab2a, 0x685ca621, 0x42638510, 0x4c6a881b, 0x5e719f06, 0x5078920d, 0x0a0fd964, 0x0406d46f, 0x161dc372, 0x1814ce79, 0x322bed48, 0x3c22e043, 0x2e39f75e, 0x2030fa55, 0xec9ab701, 0xe293ba0a, 0xf088ad17, 0xfe81a01c, 0xd4be832d, 0xdab78e26, 0xc8ac993b, 0xc6a59430, 0x9cd2df59, 0x92dbd252, 0x80c0c54f, 0x8ec9c844, 0xa4f6eb75, 0xaaffe67e, 0xb8e4f163, 0xb6edfc68, 0x0c0a67b1, 0x02036aba, 0x10187da7, 0x1e1170ac, 0x342e539d, 0x3a275e96, 0x283c498b, 0x26354480, 0x7c420fe9, 0x724b02e2, 0x605015ff, 0x6e5918f4, 0x44663bc5, 0x4a6f36ce, 0x587421d3, 0x567d2cd8, 0x37a10c7a, 0x39a80171, 0x2bb3166c, 0x25ba1b67, 0x0f853856, 0x018c355d, 0x13972240, 0x1d9e2f4b, 0x47e96422, 0x49e06929, 0x5bfb7e34, 0x55f2733f, 0x7fcd500e, 0x71c45d05, 0x63df4a18, 0x6dd64713, 0xd731dcca, 0xd938d1c1, 0xcb23c6dc, 0xc52acbd7, 0xef15e8e6, 0xe11ce5ed, 0xf307f2f0, 0xfd0efffb, 0xa779b492, 0xa970b999, 0xbb6bae84, 0xb562a38f, 0x9f5d80be, 0x91548db5, 0x834f9aa8, 0x8d4697a3 ] + U2 = [ 0x00000000, 0x0b0e090d, 0x161c121a, 0x1d121b17, 0x2c382434, 0x27362d39, 0x3a24362e, 0x312a3f23, 0x58704868, 0x537e4165, 0x4e6c5a72, 0x4562537f, 0x74486c5c, 0x7f466551, 0x62547e46, 0x695a774b, 0xb0e090d0, 0xbbee99dd, 0xa6fc82ca, 0xadf28bc7, 0x9cd8b4e4, 0x97d6bde9, 0x8ac4a6fe, 0x81caaff3, 0xe890d8b8, 0xe39ed1b5, 0xfe8ccaa2, 0xf582c3af, 0xc4a8fc8c, 0xcfa6f581, 0xd2b4ee96, 0xd9bae79b, 0x7bdb3bbb, 0x70d532b6, 0x6dc729a1, 0x66c920ac, 0x57e31f8f, 0x5ced1682, 0x41ff0d95, 0x4af10498, 0x23ab73d3, 0x28a57ade, 0x35b761c9, 0x3eb968c4, 0x0f9357e7, 0x049d5eea, 0x198f45fd, 0x12814cf0, 0xcb3bab6b, 0xc035a266, 0xdd27b971, 0xd629b07c, 0xe7038f5f, 0xec0d8652, 0xf11f9d45, 0xfa119448, 0x934be303, 0x9845ea0e, 0x8557f119, 0x8e59f814, 0xbf73c737, 0xb47dce3a, 0xa96fd52d, 0xa261dc20, 0xf6ad766d, 0xfda37f60, 0xe0b16477, 0xebbf6d7a, 0xda955259, 0xd19b5b54, 0xcc894043, 0xc787494e, 0xaedd3e05, 0xa5d33708, 0xb8c12c1f, 0xb3cf2512, 0x82e51a31, 0x89eb133c, 0x94f9082b, 0x9ff70126, 0x464de6bd, 0x4d43efb0, 0x5051f4a7, 0x5b5ffdaa, 0x6a75c289, 0x617bcb84, 0x7c69d093, 0x7767d99e, 0x1e3daed5, 0x1533a7d8, 0x0821bccf, 0x032fb5c2, 0x32058ae1, 0x390b83ec, 0x241998fb, 0x2f1791f6, 0x8d764dd6, 0x867844db, 0x9b6a5fcc, 0x906456c1, 0xa14e69e2, 0xaa4060ef, 0xb7527bf8, 0xbc5c72f5, 0xd50605be, 0xde080cb3, 0xc31a17a4, 0xc8141ea9, 0xf93e218a, 0xf2302887, 0xef223390, 0xe42c3a9d, 0x3d96dd06, 0x3698d40b, 0x2b8acf1c, 0x2084c611, 0x11aef932, 0x1aa0f03f, 0x07b2eb28, 0x0cbce225, 0x65e6956e, 0x6ee89c63, 0x73fa8774, 0x78f48e79, 0x49deb15a, 0x42d0b857, 0x5fc2a340, 0x54ccaa4d, 0xf741ecda, 0xfc4fe5d7, 0xe15dfec0, 0xea53f7cd, 0xdb79c8ee, 0xd077c1e3, 0xcd65daf4, 0xc66bd3f9, 0xaf31a4b2, 0xa43fadbf, 0xb92db6a8, 0xb223bfa5, 0x83098086, 0x8807898b, 0x9515929c, 0x9e1b9b91, 0x47a17c0a, 0x4caf7507, 0x51bd6e10, 0x5ab3671d, 0x6b99583e, 0x60975133, 0x7d854a24, 0x768b4329, 0x1fd13462, 0x14df3d6f, 0x09cd2678, 0x02c32f75, 0x33e91056, 0x38e7195b, 0x25f5024c, 0x2efb0b41, 0x8c9ad761, 0x8794de6c, 0x9a86c57b, 0x9188cc76, 0xa0a2f355, 0xabacfa58, 0xb6bee14f, 0xbdb0e842, 0xd4ea9f09, 0xdfe49604, 0xc2f68d13, 0xc9f8841e, 0xf8d2bb3d, 0xf3dcb230, 0xeecea927, 0xe5c0a02a, 0x3c7a47b1, 0x37744ebc, 0x2a6655ab, 0x21685ca6, 0x10426385, 0x1b4c6a88, 0x065e719f, 0x0d507892, 0x640a0fd9, 0x6f0406d4, 0x72161dc3, 0x791814ce, 0x48322bed, 0x433c22e0, 0x5e2e39f7, 0x552030fa, 0x01ec9ab7, 0x0ae293ba, 0x17f088ad, 0x1cfe81a0, 0x2dd4be83, 0x26dab78e, 0x3bc8ac99, 0x30c6a594, 0x599cd2df, 0x5292dbd2, 0x4f80c0c5, 0x448ec9c8, 0x75a4f6eb, 0x7eaaffe6, 0x63b8e4f1, 0x68b6edfc, 0xb10c0a67, 0xba02036a, 0xa710187d, 0xac1e1170, 0x9d342e53, 0x963a275e, 0x8b283c49, 0x80263544, 0xe97c420f, 0xe2724b02, 0xff605015, 0xf46e5918, 0xc544663b, 0xce4a6f36, 0xd3587421, 0xd8567d2c, 0x7a37a10c, 0x7139a801, 0x6c2bb316, 0x6725ba1b, 0x560f8538, 0x5d018c35, 0x40139722, 0x4b1d9e2f, 0x2247e964, 0x2949e069, 0x345bfb7e, 0x3f55f273, 0x0e7fcd50, 0x0571c45d, 0x1863df4a, 0x136dd647, 0xcad731dc, 0xc1d938d1, 0xdccb23c6, 0xd7c52acb, 0xe6ef15e8, 0xede11ce5, 0xf0f307f2, 0xfbfd0eff, 0x92a779b4, 0x99a970b9, 0x84bb6bae, 0x8fb562a3, 0xbe9f5d80, 0xb591548d, 0xa8834f9a, 0xa38d4697 ] + U3 = [ 0x00000000, 0x0d0b0e09, 0x1a161c12, 0x171d121b, 0x342c3824, 0x3927362d, 0x2e3a2436, 0x23312a3f, 0x68587048, 0x65537e41, 0x724e6c5a, 0x7f456253, 0x5c74486c, 0x517f4665, 0x4662547e, 0x4b695a77, 0xd0b0e090, 0xddbbee99, 0xcaa6fc82, 0xc7adf28b, 0xe49cd8b4, 0xe997d6bd, 0xfe8ac4a6, 0xf381caaf, 0xb8e890d8, 0xb5e39ed1, 0xa2fe8cca, 0xaff582c3, 0x8cc4a8fc, 0x81cfa6f5, 0x96d2b4ee, 0x9bd9bae7, 0xbb7bdb3b, 0xb670d532, 0xa16dc729, 0xac66c920, 0x8f57e31f, 0x825ced16, 0x9541ff0d, 0x984af104, 0xd323ab73, 0xde28a57a, 0xc935b761, 0xc43eb968, 0xe70f9357, 0xea049d5e, 0xfd198f45, 0xf012814c, 0x6bcb3bab, 0x66c035a2, 0x71dd27b9, 0x7cd629b0, 0x5fe7038f, 0x52ec0d86, 0x45f11f9d, 0x48fa1194, 0x03934be3, 0x0e9845ea, 0x198557f1, 0x148e59f8, 0x37bf73c7, 0x3ab47dce, 0x2da96fd5, 0x20a261dc, 0x6df6ad76, 0x60fda37f, 0x77e0b164, 0x7aebbf6d, 0x59da9552, 0x54d19b5b, 0x43cc8940, 0x4ec78749, 0x05aedd3e, 0x08a5d337, 0x1fb8c12c, 0x12b3cf25, 0x3182e51a, 0x3c89eb13, 0x2b94f908, 0x269ff701, 0xbd464de6, 0xb04d43ef, 0xa75051f4, 0xaa5b5ffd, 0x896a75c2, 0x84617bcb, 0x937c69d0, 0x9e7767d9, 0xd51e3dae, 0xd81533a7, 0xcf0821bc, 0xc2032fb5, 0xe132058a, 0xec390b83, 0xfb241998, 0xf62f1791, 0xd68d764d, 0xdb867844, 0xcc9b6a5f, 0xc1906456, 0xe2a14e69, 0xefaa4060, 0xf8b7527b, 0xf5bc5c72, 0xbed50605, 0xb3de080c, 0xa4c31a17, 0xa9c8141e, 0x8af93e21, 0x87f23028, 0x90ef2233, 0x9de42c3a, 0x063d96dd, 0x0b3698d4, 0x1c2b8acf, 0x112084c6, 0x3211aef9, 0x3f1aa0f0, 0x2807b2eb, 0x250cbce2, 0x6e65e695, 0x636ee89c, 0x7473fa87, 0x7978f48e, 0x5a49deb1, 0x5742d0b8, 0x405fc2a3, 0x4d54ccaa, 0xdaf741ec, 0xd7fc4fe5, 0xc0e15dfe, 0xcdea53f7, 0xeedb79c8, 0xe3d077c1, 0xf4cd65da, 0xf9c66bd3, 0xb2af31a4, 0xbfa43fad, 0xa8b92db6, 0xa5b223bf, 0x86830980, 0x8b880789, 0x9c951592, 0x919e1b9b, 0x0a47a17c, 0x074caf75, 0x1051bd6e, 0x1d5ab367, 0x3e6b9958, 0x33609751, 0x247d854a, 0x29768b43, 0x621fd134, 0x6f14df3d, 0x7809cd26, 0x7502c32f, 0x5633e910, 0x5b38e719, 0x4c25f502, 0x412efb0b, 0x618c9ad7, 0x6c8794de, 0x7b9a86c5, 0x769188cc, 0x55a0a2f3, 0x58abacfa, 0x4fb6bee1, 0x42bdb0e8, 0x09d4ea9f, 0x04dfe496, 0x13c2f68d, 0x1ec9f884, 0x3df8d2bb, 0x30f3dcb2, 0x27eecea9, 0x2ae5c0a0, 0xb13c7a47, 0xbc37744e, 0xab2a6655, 0xa621685c, 0x85104263, 0x881b4c6a, 0x9f065e71, 0x920d5078, 0xd9640a0f, 0xd46f0406, 0xc372161d, 0xce791814, 0xed48322b, 0xe0433c22, 0xf75e2e39, 0xfa552030, 0xb701ec9a, 0xba0ae293, 0xad17f088, 0xa01cfe81, 0x832dd4be, 0x8e26dab7, 0x993bc8ac, 0x9430c6a5, 0xdf599cd2, 0xd25292db, 0xc54f80c0, 0xc8448ec9, 0xeb75a4f6, 0xe67eaaff, 0xf163b8e4, 0xfc68b6ed, 0x67b10c0a, 0x6aba0203, 0x7da71018, 0x70ac1e11, 0x539d342e, 0x5e963a27, 0x498b283c, 0x44802635, 0x0fe97c42, 0x02e2724b, 0x15ff6050, 0x18f46e59, 0x3bc54466, 0x36ce4a6f, 0x21d35874, 0x2cd8567d, 0x0c7a37a1, 0x017139a8, 0x166c2bb3, 0x1b6725ba, 0x38560f85, 0x355d018c, 0x22401397, 0x2f4b1d9e, 0x642247e9, 0x692949e0, 0x7e345bfb, 0x733f55f2, 0x500e7fcd, 0x5d0571c4, 0x4a1863df, 0x47136dd6, 0xdccad731, 0xd1c1d938, 0xc6dccb23, 0xcbd7c52a, 0xe8e6ef15, 0xe5ede11c, 0xf2f0f307, 0xfffbfd0e, 0xb492a779, 0xb999a970, 0xae84bb6b, 0xa38fb562, 0x80be9f5d, 0x8db59154, 0x9aa8834f, 0x97a38d46 ] + U4 = [ 0x00000000, 0x090d0b0e, 0x121a161c, 0x1b171d12, 0x24342c38, 0x2d392736, 0x362e3a24, 0x3f23312a, 0x48685870, 0x4165537e, 0x5a724e6c, 0x537f4562, 0x6c5c7448, 0x65517f46, 0x7e466254, 0x774b695a, 0x90d0b0e0, 0x99ddbbee, 0x82caa6fc, 0x8bc7adf2, 0xb4e49cd8, 0xbde997d6, 0xa6fe8ac4, 0xaff381ca, 0xd8b8e890, 0xd1b5e39e, 0xcaa2fe8c, 0xc3aff582, 0xfc8cc4a8, 0xf581cfa6, 0xee96d2b4, 0xe79bd9ba, 0x3bbb7bdb, 0x32b670d5, 0x29a16dc7, 0x20ac66c9, 0x1f8f57e3, 0x16825ced, 0x0d9541ff, 0x04984af1, 0x73d323ab, 0x7ade28a5, 0x61c935b7, 0x68c43eb9, 0x57e70f93, 0x5eea049d, 0x45fd198f, 0x4cf01281, 0xab6bcb3b, 0xa266c035, 0xb971dd27, 0xb07cd629, 0x8f5fe703, 0x8652ec0d, 0x9d45f11f, 0x9448fa11, 0xe303934b, 0xea0e9845, 0xf1198557, 0xf8148e59, 0xc737bf73, 0xce3ab47d, 0xd52da96f, 0xdc20a261, 0x766df6ad, 0x7f60fda3, 0x6477e0b1, 0x6d7aebbf, 0x5259da95, 0x5b54d19b, 0x4043cc89, 0x494ec787, 0x3e05aedd, 0x3708a5d3, 0x2c1fb8c1, 0x2512b3cf, 0x1a3182e5, 0x133c89eb, 0x082b94f9, 0x01269ff7, 0xe6bd464d, 0xefb04d43, 0xf4a75051, 0xfdaa5b5f, 0xc2896a75, 0xcb84617b, 0xd0937c69, 0xd99e7767, 0xaed51e3d, 0xa7d81533, 0xbccf0821, 0xb5c2032f, 0x8ae13205, 0x83ec390b, 0x98fb2419, 0x91f62f17, 0x4dd68d76, 0x44db8678, 0x5fcc9b6a, 0x56c19064, 0x69e2a14e, 0x60efaa40, 0x7bf8b752, 0x72f5bc5c, 0x05bed506, 0x0cb3de08, 0x17a4c31a, 0x1ea9c814, 0x218af93e, 0x2887f230, 0x3390ef22, 0x3a9de42c, 0xdd063d96, 0xd40b3698, 0xcf1c2b8a, 0xc6112084, 0xf93211ae, 0xf03f1aa0, 0xeb2807b2, 0xe2250cbc, 0x956e65e6, 0x9c636ee8, 0x877473fa, 0x8e7978f4, 0xb15a49de, 0xb85742d0, 0xa3405fc2, 0xaa4d54cc, 0xecdaf741, 0xe5d7fc4f, 0xfec0e15d, 0xf7cdea53, 0xc8eedb79, 0xc1e3d077, 0xdaf4cd65, 0xd3f9c66b, 0xa4b2af31, 0xadbfa43f, 0xb6a8b92d, 0xbfa5b223, 0x80868309, 0x898b8807, 0x929c9515, 0x9b919e1b, 0x7c0a47a1, 0x75074caf, 0x6e1051bd, 0x671d5ab3, 0x583e6b99, 0x51336097, 0x4a247d85, 0x4329768b, 0x34621fd1, 0x3d6f14df, 0x267809cd, 0x2f7502c3, 0x105633e9, 0x195b38e7, 0x024c25f5, 0x0b412efb, 0xd7618c9a, 0xde6c8794, 0xc57b9a86, 0xcc769188, 0xf355a0a2, 0xfa58abac, 0xe14fb6be, 0xe842bdb0, 0x9f09d4ea, 0x9604dfe4, 0x8d13c2f6, 0x841ec9f8, 0xbb3df8d2, 0xb230f3dc, 0xa927eece, 0xa02ae5c0, 0x47b13c7a, 0x4ebc3774, 0x55ab2a66, 0x5ca62168, 0x63851042, 0x6a881b4c, 0x719f065e, 0x78920d50, 0x0fd9640a, 0x06d46f04, 0x1dc37216, 0x14ce7918, 0x2bed4832, 0x22e0433c, 0x39f75e2e, 0x30fa5520, 0x9ab701ec, 0x93ba0ae2, 0x88ad17f0, 0x81a01cfe, 0xbe832dd4, 0xb78e26da, 0xac993bc8, 0xa59430c6, 0xd2df599c, 0xdbd25292, 0xc0c54f80, 0xc9c8448e, 0xf6eb75a4, 0xffe67eaa, 0xe4f163b8, 0xedfc68b6, 0x0a67b10c, 0x036aba02, 0x187da710, 0x1170ac1e, 0x2e539d34, 0x275e963a, 0x3c498b28, 0x35448026, 0x420fe97c, 0x4b02e272, 0x5015ff60, 0x5918f46e, 0x663bc544, 0x6f36ce4a, 0x7421d358, 0x7d2cd856, 0xa10c7a37, 0xa8017139, 0xb3166c2b, 0xba1b6725, 0x8538560f, 0x8c355d01, 0x97224013, 0x9e2f4b1d, 0xe9642247, 0xe0692949, 0xfb7e345b, 0xf2733f55, 0xcd500e7f, 0xc45d0571, 0xdf4a1863, 0xd647136d, 0x31dccad7, 0x38d1c1d9, 0x23c6dccb, 0x2acbd7c5, 0x15e8e6ef, 0x1ce5ede1, 0x07f2f0f3, 0x0efffbfd, 0x79b492a7, 0x70b999a9, 0x6bae84bb, 0x62a38fb5, 0x5d80be9f, 0x548db591, 0x4f9aa883, 0x4697a38d ] + + def __init__(self, key): + + if len(key) not in (16, 24, 32): + raise ValueError('Invalid key size') + + rounds = self.number_of_rounds[len(key)] + + # Encryption round keys + self._Ke = [[0] * 4 for i in xrange(rounds + 1)] + + # Decryption round keys + self._Kd = [[0] * 4 for i in xrange(rounds + 1)] + + round_key_count = (rounds + 1) * 4 + KC = len(key) // 4 + + # Convert the key into ints + tk = [ struct.unpack('>i', key[i:i + 4])[0] for i in xrange(0, len(key), 4) ] + + # Copy values into round key arrays + for i in xrange(0, KC): + self._Ke[i // 4][i % 4] = tk[i] + self._Kd[rounds - (i // 4)][i % 4] = tk[i] + + # Key expansion (fips-197 section 5.2) + rconpointer = 0 + t = KC + while t < round_key_count: + + tt = tk[KC - 1] + tk[0] ^= ((self.S[(tt >> 16) & 0xFF] << 24) ^ + (self.S[(tt >> 8) & 0xFF] << 16) ^ + (self.S[ tt & 0xFF] << 8) ^ + self.S[(tt >> 24) & 0xFF] ^ + (self.rcon[rconpointer] << 24)) + rconpointer += 1 + + if KC != 8: + for i in xrange(1, KC): + tk[i] ^= tk[i - 1] + + # Key expansion for 256-bit keys is "slightly different" (fips-197) + else: + for i in xrange(1, KC // 2): + tk[i] ^= tk[i - 1] + tt = tk[KC // 2 - 1] + + tk[KC // 2] ^= (self.S[ tt & 0xFF] ^ + (self.S[(tt >> 8) & 0xFF] << 8) ^ + (self.S[(tt >> 16) & 0xFF] << 16) ^ + (self.S[(tt >> 24) & 0xFF] << 24)) + + for i in xrange(KC // 2 + 1, KC): + tk[i] ^= tk[i - 1] + + # Copy values into round key arrays + j = 0 + while j < KC and t < round_key_count: + self._Ke[t // 4][t % 4] = tk[j] + self._Kd[rounds - (t // 4)][t % 4] = tk[j] + j += 1 + t += 1 + + # Inverse-Cipher-ify the decryption round key (fips-197 section 5.3) + for r in xrange(1, rounds): + for j in xrange(0, 4): + tt = self._Kd[r][j] + self._Kd[r][j] = (self.U1[(tt >> 24) & 0xFF] ^ + self.U2[(tt >> 16) & 0xFF] ^ + self.U3[(tt >> 8) & 0xFF] ^ + self.U4[ tt & 0xFF]) + + def encrypt(self, plaintext): + 'Encrypt a block of plain text using the AES block cipher.' + + if len(plaintext) != 16: + raise ValueError('wrong block length') + + rounds = len(self._Ke) - 1 + (s1, s2, s3) = [1, 2, 3] + a = [0, 0, 0, 0] + + # Convert plaintext to (ints ^ key) + t = [(_compact_word(plaintext[4 * i:4 * i + 4]) ^ self._Ke[0][i]) for i in xrange(0, 4)] + + # Apply round transforms + for r in xrange(1, rounds): + for i in xrange(0, 4): + a[i] = (self.T1[(t[ i ] >> 24) & 0xFF] ^ + self.T2[(t[(i + s1) % 4] >> 16) & 0xFF] ^ + self.T3[(t[(i + s2) % 4] >> 8) & 0xFF] ^ + self.T4[ t[(i + s3) % 4] & 0xFF] ^ + self._Ke[r][i]) + t = copy.copy(a) + + # The last round is special + result = [ ] + for i in xrange(0, 4): + tt = self._Ke[rounds][i] + result.append((self.S[(t[ i ] >> 24) & 0xFF] ^ (tt >> 24)) & 0xFF) + result.append((self.S[(t[(i + s1) % 4] >> 16) & 0xFF] ^ (tt >> 16)) & 0xFF) + result.append((self.S[(t[(i + s2) % 4] >> 8) & 0xFF] ^ (tt >> 8)) & 0xFF) + result.append((self.S[ t[(i + s3) % 4] & 0xFF] ^ tt ) & 0xFF) + + return result + + def decrypt(self, ciphertext): + 'Decrypt a block of cipher text using the AES block cipher.' + + if len(ciphertext) != 16: + raise ValueError('wrong block length') + + rounds = len(self._Kd) - 1 + (s1, s2, s3) = [3, 2, 1] + a = [0, 0, 0, 0] + + # Convert ciphertext to (ints ^ key) + t = [(_compact_word(ciphertext[4 * i:4 * i + 4]) ^ self._Kd[0][i]) for i in xrange(0, 4)] + + # Apply round transforms + for r in xrange(1, rounds): + for i in xrange(0, 4): + a[i] = (self.T5[(t[ i ] >> 24) & 0xFF] ^ + self.T6[(t[(i + s1) % 4] >> 16) & 0xFF] ^ + self.T7[(t[(i + s2) % 4] >> 8) & 0xFF] ^ + self.T8[ t[(i + s3) % 4] & 0xFF] ^ + self._Kd[r][i]) + t = copy.copy(a) + + # The last round is special + result = [ ] + for i in xrange(0, 4): + tt = self._Kd[rounds][i] + result.append((self.Si[(t[ i ] >> 24) & 0xFF] ^ (tt >> 24)) & 0xFF) + result.append((self.Si[(t[(i + s1) % 4] >> 16) & 0xFF] ^ (tt >> 16)) & 0xFF) + result.append((self.Si[(t[(i + s2) % 4] >> 8) & 0xFF] ^ (tt >> 8)) & 0xFF) + result.append((self.Si[ t[(i + s3) % 4] & 0xFF] ^ tt ) & 0xFF) + + return result + + +class Counter(object): + '''A counter object for the Counter (CTR) mode of operation. + + To create a custom counter, you can usually just override the + increment method.''' + + def __init__(self, initial_value = 1): + + # Convert the value into an array of bytes long + self._counter = [ ((initial_value >> i) % 256) for i in xrange(128 - 8, -1, -8) ] + + value = property(lambda s: s._counter) + + def increment(self): + '''Increment the counter (overflow rolls back to 0).''' + + for i in xrange(len(self._counter) - 1, -1, -1): + self._counter[i] += 1 + + if self._counter[i] < 256: break + + # Carry the one + self._counter[i] = 0 + + # Overflow + else: + self._counter = [ 0 ] * len(self._counter) + + +class AESBlockModeOfOperation(object): + '''Super-class for AES modes of operation that require blocks.''' + def __init__(self, key): + self._aes = AES(key) + + def decrypt(self, ciphertext): + raise Exception('not implemented') + + def encrypt(self, plaintext): + raise Exception('not implemented') + + +class AESStreamModeOfOperation(AESBlockModeOfOperation): + '''Super-class for AES modes of operation that are stream-ciphers.''' + +class AESSegmentModeOfOperation(AESStreamModeOfOperation): + '''Super-class for AES modes of operation that segment data.''' + + segment_bytes = 16 + + + +class AESModeOfOperationECB(AESBlockModeOfOperation): + '''AES Electronic Codebook Mode of Operation. + + o Block-cipher, so data must be padded to 16 byte boundaries + + Security Notes: + o This mode is not recommended + o Any two identical blocks produce identical encrypted values, + exposing data patterns. (See the image of Tux on wikipedia) + + Also see: + o https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_.28ECB.29 + o See NIST SP800-38A (http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf); section 6.1''' + + + name = "Electronic Codebook (ECB)" + + def encrypt(self, plaintext): + if len(plaintext) != 16: + raise ValueError('plaintext block must be 16 bytes') + + plaintext = _string_to_bytes(plaintext) + return _bytes_to_string(self._aes.encrypt(plaintext)) + + def decrypt(self, ciphertext): + if len(ciphertext) != 16: + raise ValueError('ciphertext block must be 16 bytes') + + ciphertext = _string_to_bytes(ciphertext) + return _bytes_to_string(self._aes.decrypt(ciphertext)) + + + +class AESModeOfOperationCBC(AESBlockModeOfOperation): + '''AES Cipher-Block Chaining Mode of Operation. + + o The Initialization Vector (IV) + o Block-cipher, so data must be padded to 16 byte boundaries + o An incorrect initialization vector will only cause the first + block to be corrupt; all other blocks will be intact + o A corrupt bit in the cipher text will cause a block to be + corrupted, and the next block to be inverted, but all other + blocks will be intact. + + Security Notes: + o This method (and CTR) ARE recommended. + + Also see: + o https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher-block_chaining_.28CBC.29 + o See NIST SP800-38A (http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf); section 6.2''' + + + name = "Cipher-Block Chaining (CBC)" + + def __init__(self, key, iv = None): + if iv is None: + self._last_cipherblock = [ 0 ] * 16 + elif len(iv) != 16: + raise ValueError('initialization vector must be 16 bytes') + else: + self._last_cipherblock = _string_to_bytes(iv) + + AESBlockModeOfOperation.__init__(self, key) + + def encrypt(self, plaintext): + if len(plaintext) != 16: + raise ValueError('plaintext block must be 16 bytes') + + plaintext = _string_to_bytes(plaintext) + precipherblock = [ (p ^ l) for (p, l) in zip(plaintext, self._last_cipherblock) ] + self._last_cipherblock = self._aes.encrypt(precipherblock) + + return _bytes_to_string(self._last_cipherblock) + + def decrypt(self, ciphertext): + if len(ciphertext) != 16: + raise ValueError('ciphertext block must be 16 bytes') + + cipherblock = _string_to_bytes(ciphertext) + plaintext = [ (p ^ l) for (p, l) in zip(self._aes.decrypt(cipherblock), self._last_cipherblock) ] + self._last_cipherblock = cipherblock + + return _bytes_to_string(plaintext) + + + +class AESModeOfOperationCFB(AESSegmentModeOfOperation): + '''AES Cipher Feedback Mode of Operation. + + o A stream-cipher, so input does not need to be padded to blocks, + but does need to be padded to segment_size + + Also see: + o https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_.28CFB.29 + o See NIST SP800-38A (http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf); section 6.3''' + + + name = "Cipher Feedback (CFB)" + + def __init__(self, key, iv, segment_size = 1): + if segment_size == 0: segment_size = 1 + + if iv is None: + self._shift_register = [ 0 ] * 16 + elif len(iv) != 16: + raise ValueError('initialization vector must be 16 bytes') + else: + self._shift_register = _string_to_bytes(iv) + + self._segment_bytes = segment_size + + AESBlockModeOfOperation.__init__(self, key) + + segment_bytes = property(lambda s: s._segment_bytes) + + def encrypt(self, plaintext): + if len(plaintext) % self._segment_bytes != 0: + raise ValueError('plaintext block must be a multiple of segment_size') + + plaintext = _string_to_bytes(plaintext) + + # Break block into segments + encrypted = [ ] + for i in xrange(0, len(plaintext), self._segment_bytes): + plaintext_segment = plaintext[i: i + self._segment_bytes] + xor_segment = self._aes.encrypt(self._shift_register)[:len(plaintext_segment)] + cipher_segment = [ (p ^ x) for (p, x) in zip(plaintext_segment, xor_segment) ] + + # Shift the top bits out and the ciphertext in + self._shift_register = _concat_list(self._shift_register[len(cipher_segment):], cipher_segment) + + encrypted.extend(cipher_segment) + + return _bytes_to_string(encrypted) + + def decrypt(self, ciphertext): + if len(ciphertext) % self._segment_bytes != 0: + raise ValueError('ciphertext block must be a multiple of segment_size') + + ciphertext = _string_to_bytes(ciphertext) + + # Break block into segments + decrypted = [ ] + for i in xrange(0, len(ciphertext), self._segment_bytes): + cipher_segment = ciphertext[i: i + self._segment_bytes] + xor_segment = self._aes.encrypt(self._shift_register)[:len(cipher_segment)] + plaintext_segment = [ (p ^ x) for (p, x) in zip(cipher_segment, xor_segment) ] + + # Shift the top bits out and the ciphertext in + self._shift_register = _concat_list(self._shift_register[len(cipher_segment):], cipher_segment) + + decrypted.extend(plaintext_segment) + + return _bytes_to_string(decrypted) + + + +class AESModeOfOperationOFB(AESStreamModeOfOperation): + '''AES Output Feedback Mode of Operation. + + o A stream-cipher, so input does not need to be padded to blocks, + allowing arbitrary length data. + o A bit twiddled in the cipher text, twiddles the same bit in the + same bit in the plain text, which can be useful for error + correction techniques. + + Also see: + o https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Output_feedback_.28OFB.29 + o See NIST SP800-38A (http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf); section 6.4''' + + + name = "Output Feedback (OFB)" + + def __init__(self, key, iv = None): + if iv is None: + self._last_precipherblock = [ 0 ] * 16 + elif len(iv) != 16: + raise ValueError('initialization vector must be 16 bytes') + else: + self._last_precipherblock = _string_to_bytes(iv) + + self._remaining_block = [ ] + + AESBlockModeOfOperation.__init__(self, key) + + def encrypt(self, plaintext): + encrypted = [ ] + for p in _string_to_bytes(plaintext): + if len(self._remaining_block) == 0: + self._remaining_block = self._aes.encrypt(self._last_precipherblock) + self._last_precipherblock = [ ] + precipherbyte = self._remaining_block.pop(0) + self._last_precipherblock.append(precipherbyte) + cipherbyte = p ^ precipherbyte + encrypted.append(cipherbyte) + + return _bytes_to_string(encrypted) + + def decrypt(self, ciphertext): + # AES-OFB is symetric + return self.encrypt(ciphertext) + + + +class AESModeOfOperationCTR(AESStreamModeOfOperation): + '''AES Counter Mode of Operation. + + o A stream-cipher, so input does not need to be padded to blocks, + allowing arbitrary length data. + o The counter must be the same size as the key size (ie. len(key)) + o Each block independant of the other, so a corrupt byte will not + damage future blocks. + o Each block has a uniue counter value associated with it, which + contributes to the encrypted value, so no data patterns are + leaked. + o Also known as: Counter Mode (CM), Integer Counter Mode (ICM) and + Segmented Integer Counter (SIC + + Security Notes: + o This method (and CBC) ARE recommended. + o Each message block is associated with a counter value which must be + unique for ALL messages with the same key. Otherwise security may be + compromised. + + Also see: + + o https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29 + o See NIST SP800-38A (http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf); section 6.5 + and Appendix B for managing the initial counter''' + + + name = "Counter (CTR)" + + def __init__(self, key, counter = None): + AESBlockModeOfOperation.__init__(self, key) + + if counter is None: + counter = Counter() + + self._counter = counter + self._remaining_counter = [ ] + + def encrypt(self, plaintext): + while len(self._remaining_counter) < len(plaintext): + self._remaining_counter += self._aes.encrypt(self._counter.value) + self._counter.increment() + + plaintext = _string_to_bytes(plaintext) + + encrypted = [ (p ^ c) for (p, c) in zip(plaintext, self._remaining_counter) ] + self._remaining_counter = self._remaining_counter[len(encrypted):] + + return _bytes_to_string(encrypted) + + def decrypt(self, crypttext): + # AES-CTR is symetric + return self.encrypt(crypttext) + + +# Simple lookup table for each mode +AESModesOfOperation = dict( + ctr = AESModeOfOperationCTR, + cbc = AESModeOfOperationCBC, + cfb = AESModeOfOperationCFB, + ecb = AESModeOfOperationECB, + ofb = AESModeOfOperationOFB, +) diff --git a/src/lib/pyaes/blockfeeder.py b/src/lib/pyaes/blockfeeder.py new file mode 100644 index 00000000..b9a904d2 --- /dev/null +++ b/src/lib/pyaes/blockfeeder.py @@ -0,0 +1,227 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014 Richard Moore +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +from .aes import AESBlockModeOfOperation, AESSegmentModeOfOperation, AESStreamModeOfOperation +from .util import append_PKCS7_padding, strip_PKCS7_padding, to_bufferable + + +# First we inject three functions to each of the modes of operations +# +# _can_consume(size) +# - Given a size, determine how many bytes could be consumed in +# a single call to either the decrypt or encrypt method +# +# _final_encrypt(data, padding = PADDING_DEFAULT) +# - call and return encrypt on this (last) chunk of data, +# padding as necessary; this will always be at least 16 +# bytes unless the total incoming input was less than 16 +# bytes +# +# _final_decrypt(data, padding = PADDING_DEFAULT) +# - same as _final_encrypt except for decrypt, for +# stripping off padding +# + +PADDING_NONE = 'none' +PADDING_DEFAULT = 'default' + +# @TODO: Ciphertext stealing and explicit PKCS#7 +# PADDING_CIPHERTEXT_STEALING +# PADDING_PKCS7 + +# ECB and CBC are block-only ciphers + +def _block_can_consume(self, size): + if size >= 16: return 16 + return 0 + +# After padding, we may have more than one block +def _block_final_encrypt(self, data, padding = PADDING_DEFAULT): + if padding == PADDING_DEFAULT: + data = append_PKCS7_padding(data) + + elif padding == PADDING_NONE: + if len(data) != 16: + raise Exception('invalid data length for final block') + else: + raise Exception('invalid padding option') + + if len(data) == 32: + return self.encrypt(data[:16]) + self.encrypt(data[16:]) + + return self.encrypt(data) + + +def _block_final_decrypt(self, data, padding = PADDING_DEFAULT): + if padding == PADDING_DEFAULT: + return strip_PKCS7_padding(self.decrypt(data)) + + if padding == PADDING_NONE: + if len(data) != 16: + raise Exception('invalid data length for final block') + return self.decrypt(data) + + raise Exception('invalid padding option') + +AESBlockModeOfOperation._can_consume = _block_can_consume +AESBlockModeOfOperation._final_encrypt = _block_final_encrypt +AESBlockModeOfOperation._final_decrypt = _block_final_decrypt + + + +# CFB is a segment cipher + +def _segment_can_consume(self, size): + return self.segment_bytes * int(size // self.segment_bytes) + +# CFB can handle a non-segment-sized block at the end using the remaining cipherblock +def _segment_final_encrypt(self, data, padding = PADDING_DEFAULT): + if padding != PADDING_DEFAULT: + raise Exception('invalid padding option') + + faux_padding = (chr(0) * (self.segment_bytes - (len(data) % self.segment_bytes))) + padded = data + to_bufferable(faux_padding) + return self.encrypt(padded)[:len(data)] + +# CFB can handle a non-segment-sized block at the end using the remaining cipherblock +def _segment_final_decrypt(self, data, padding = PADDING_DEFAULT): + if padding != PADDING_DEFAULT: + raise Exception('invalid padding option') + + faux_padding = (chr(0) * (self.segment_bytes - (len(data) % self.segment_bytes))) + padded = data + to_bufferable(faux_padding) + return self.decrypt(padded)[:len(data)] + +AESSegmentModeOfOperation._can_consume = _segment_can_consume +AESSegmentModeOfOperation._final_encrypt = _segment_final_encrypt +AESSegmentModeOfOperation._final_decrypt = _segment_final_decrypt + + + +# OFB and CTR are stream ciphers + +def _stream_can_consume(self, size): + return size + +def _stream_final_encrypt(self, data, padding = PADDING_DEFAULT): + if padding not in [PADDING_NONE, PADDING_DEFAULT]: + raise Exception('invalid padding option') + + return self.encrypt(data) + +def _stream_final_decrypt(self, data, padding = PADDING_DEFAULT): + if padding not in [PADDING_NONE, PADDING_DEFAULT]: + raise Exception('invalid padding option') + + return self.decrypt(data) + +AESStreamModeOfOperation._can_consume = _stream_can_consume +AESStreamModeOfOperation._final_encrypt = _stream_final_encrypt +AESStreamModeOfOperation._final_decrypt = _stream_final_decrypt + + + +class BlockFeeder(object): + '''The super-class for objects to handle chunking a stream of bytes + into the appropriate block size for the underlying mode of operation + and applying (or stripping) padding, as necessary.''' + + def __init__(self, mode, feed, final, padding = PADDING_DEFAULT): + self._mode = mode + self._feed = feed + self._final = final + self._buffer = to_bufferable("") + self._padding = padding + + def feed(self, data = None): + '''Provide bytes to encrypt (or decrypt), returning any bytes + possible from this or any previous calls to feed. + + Call with None or an empty string to flush the mode of + operation and return any final bytes; no further calls to + feed may be made.''' + + if self._buffer is None: + raise ValueError('already finished feeder') + + # Finalize; process the spare bytes we were keeping + if data is None: + result = self._final(self._buffer, self._padding) + self._buffer = None + return result + + self._buffer += to_bufferable(data) + + # We keep 16 bytes around so we can determine padding + result = to_bufferable('') + while len(self._buffer) > 16: + can_consume = self._mode._can_consume(len(self._buffer) - 16) + if can_consume == 0: break + result += self._feed(self._buffer[:can_consume]) + self._buffer = self._buffer[can_consume:] + + return result + + +class Encrypter(BlockFeeder): + 'Accepts bytes of plaintext and returns encrypted ciphertext.' + + def __init__(self, mode, padding = PADDING_DEFAULT): + BlockFeeder.__init__(self, mode, mode.encrypt, mode._final_encrypt, padding) + + +class Decrypter(BlockFeeder): + 'Accepts bytes of ciphertext and returns decrypted plaintext.' + + def __init__(self, mode, padding = PADDING_DEFAULT): + BlockFeeder.__init__(self, mode, mode.decrypt, mode._final_decrypt, padding) + + +# 8kb blocks +BLOCK_SIZE = (1 << 13) + +def _feed_stream(feeder, in_stream, out_stream, block_size = BLOCK_SIZE): + 'Uses feeder to read and convert from in_stream and write to out_stream.' + + while True: + chunk = in_stream.read(block_size) + if not chunk: + break + converted = feeder.feed(chunk) + out_stream.write(converted) + converted = feeder.feed() + out_stream.write(converted) + + +def encrypt_stream(mode, in_stream, out_stream, block_size = BLOCK_SIZE, padding = PADDING_DEFAULT): + 'Encrypts a stream of bytes from in_stream to out_stream using mode.' + + encrypter = Encrypter(mode, padding = padding) + _feed_stream(encrypter, in_stream, out_stream, block_size) + + +def decrypt_stream(mode, in_stream, out_stream, block_size = BLOCK_SIZE, padding = PADDING_DEFAULT): + 'Decrypts a stream of bytes from in_stream to out_stream using mode.' + + decrypter = Decrypter(mode, padding = padding) + _feed_stream(decrypter, in_stream, out_stream, block_size) diff --git a/src/lib/pyaes/util.py b/src/lib/pyaes/util.py new file mode 100644 index 00000000..081a3759 --- /dev/null +++ b/src/lib/pyaes/util.py @@ -0,0 +1,60 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014 Richard Moore +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# Why to_bufferable? +# Python 3 is very different from Python 2.x when it comes to strings of text +# and strings of bytes; in Python 3, strings of bytes do not exist, instead to +# represent arbitrary binary data, we must use the "bytes" object. This method +# ensures the object behaves as we need it to. + +def to_bufferable(binary): + return binary + +def _get_byte(c): + return ord(c) + +try: + xrange +except: + + def to_bufferable(binary): + if isinstance(binary, bytes): + return binary + return bytes(ord(b) for b in binary) + + def _get_byte(c): + return c + +def append_PKCS7_padding(data): + pad = 16 - (len(data) % 16) + return data + to_bufferable(chr(pad) * pad) + +def strip_PKCS7_padding(data): + if len(data) % 16 != 0: + raise ValueError("invalid length") + + pad = _get_byte(data[-1]) + + if pad > 16: + raise ValueError("invalid padding byte") + + return data[:-pad] diff --git a/src/lib/sslcrypto/LICENSE b/src/lib/sslcrypto/LICENSE new file mode 100644 index 00000000..2feefc45 --- /dev/null +++ b/src/lib/sslcrypto/LICENSE @@ -0,0 +1,27 @@ +MIT License + +Copyright (c) 2019 Ivan Machugovskiy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Additionally, the following licenses must be preserved: + +- ripemd implementation is licensed under BSD-3 by Markus Friedl, see `_ripemd.py`; +- jacobian curve implementation is dual-licensed under MIT or public domain license, see `_jacobian.py`. diff --git a/src/lib/sslcrypto/__init__.py b/src/lib/sslcrypto/__init__.py new file mode 100644 index 00000000..77f9b3f3 --- /dev/null +++ b/src/lib/sslcrypto/__init__.py @@ -0,0 +1,6 @@ +__all__ = ["aes", "ecc", "rsa"] + +try: + from .openssl import aes, ecc, rsa +except OSError: + from .fallback import aes, ecc, rsa diff --git a/src/lib/sslcrypto/_aes.py b/src/lib/sslcrypto/_aes.py new file mode 100644 index 00000000..4f8d4ec2 --- /dev/null +++ b/src/lib/sslcrypto/_aes.py @@ -0,0 +1,53 @@ +# pylint: disable=import-outside-toplevel + +class AES: + def __init__(self, backend, fallback=None): + self._backend = backend + self._fallback = fallback + + + def get_algo_key_length(self, algo): + if algo.count("-") != 2: + raise ValueError("Invalid algorithm name") + try: + return int(algo.split("-")[1]) // 8 + except ValueError: + raise ValueError("Invalid algorithm name") from None + + + def new_key(self, algo="aes-256-cbc"): + if not self._backend.is_algo_supported(algo): + if self._fallback is None: + raise ValueError("This algorithm is not supported") + return self._fallback.new_key(algo) + return self._backend.random(self.get_algo_key_length(algo)) + + + def encrypt(self, data, key, algo="aes-256-cbc"): + if not self._backend.is_algo_supported(algo): + if self._fallback is None: + raise ValueError("This algorithm is not supported") + return self._fallback.encrypt(data, key, algo) + + key_length = self.get_algo_key_length(algo) + if len(key) != key_length: + raise ValueError("Expected key to be {} bytes, got {} bytes".format(key_length, len(key))) + + return self._backend.encrypt(data, key, algo) + + + def decrypt(self, ciphertext, iv, key, algo="aes-256-cbc"): + if not self._backend.is_algo_supported(algo): + if self._fallback is None: + raise ValueError("This algorithm is not supported") + return self._fallback.decrypt(ciphertext, iv, key, algo) + + key_length = self.get_algo_key_length(algo) + if len(key) != key_length: + raise ValueError("Expected key to be {} bytes, got {} bytes".format(key_length, len(key))) + + return self._backend.decrypt(ciphertext, iv, key, algo) + + + def get_backend(self): + return self._backend.get_backend() diff --git a/src/lib/sslcrypto/_ecc.py b/src/lib/sslcrypto/_ecc.py new file mode 100644 index 00000000..88e04576 --- /dev/null +++ b/src/lib/sslcrypto/_ecc.py @@ -0,0 +1,506 @@ +import hashlib +import struct +import hmac +import base58 + + +try: + hashlib.new("ripemd160") +except ValueError: + # No native implementation + from . import _ripemd + def ripemd160(*args): + return _ripemd.new(*args) +else: + # Use OpenSSL + def ripemd160(*args): + return hashlib.new("ripemd160", *args) + + +class ECC: + # pylint: disable=line-too-long + # name: (nid, p, n, a, b, (Gx, Gy)), + CURVES = { + "secp112r1": ( + 704, + 0xDB7C2ABF62E35E668076BEAD208B, + 0xDB7C2ABF62E35E7628DFAC6561C5, + 0xDB7C2ABF62E35E668076BEAD2088, + 0x659EF8BA043916EEDE8911702B22, + ( + 0x09487239995A5EE76B55F9C2F098, + 0xA89CE5AF8724C0A23E0E0FF77500 + ) + ), + "secp112r2": ( + 705, + 0xDB7C2ABF62E35E668076BEAD208B, + 0x36DF0AAFD8B8D7597CA10520D04B, + 0x6127C24C05F38A0AAAF65C0EF02C, + 0x51DEF1815DB5ED74FCC34C85D709, + ( + 0x4BA30AB5E892B4E1649DD0928643, + 0xADCD46F5882E3747DEF36E956E97 + ) + ), + "secp128r1": ( + 706, + 0xFFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFF, + 0xFFFFFFFE0000000075A30D1B9038A115, + 0xFFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFC, + 0xE87579C11079F43DD824993C2CEE5ED3, + ( + 0x161FF7528B899B2D0C28607CA52C5B86, + 0xCF5AC8395BAFEB13C02DA292DDED7A83 + ) + ), + "secp128r2": ( + 707, + 0xFFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFF, + 0x3FFFFFFF7FFFFFFFBE0024720613B5A3, + 0xD6031998D1B3BBFEBF59CC9BBFF9AEE1, + 0x5EEEFCA380D02919DC2C6558BB6D8A5D, + ( + 0x7B6AA5D85E572983E6FB32A7CDEBC140, + 0x27B6916A894D3AEE7106FE805FC34B44 + ) + ), + "secp160k1": ( + 708, + 0x00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFAC73, + 0x0100000000000000000001B8FA16DFAB9ACA16B6B3, + 0, + 7, + ( + 0x3B4C382CE37AA192A4019E763036F4F5DD4D7EBB, + 0x938CF935318FDCED6BC28286531733C3F03C4FEE + ) + ), + "secp160r1": ( + 709, + 0x00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFFFFFF, + 0x0100000000000000000001F4C8F927AED3CA752257, + 0x00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFFFFFC, + 0x001C97BEFC54BD7A8B65ACF89F81D4D4ADC565FA45, + ( + 0x4A96B5688EF573284664698968C38BB913CBFC82, + 0x23A628553168947D59DCC912042351377AC5FB32 + ) + ), + "secp160r2": ( + 710, + 0x00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFAC73, + 0x0100000000000000000000351EE786A818F3A1A16B, + 0x00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFAC70, + 0x00B4E134D3FB59EB8BAB57274904664D5AF50388BA, + ( + 0x52DCB034293A117E1F4FF11B30F7199D3144CE6D, + 0xFEAFFEF2E331F296E071FA0DF9982CFEA7D43F2E + ) + ), + "secp192k1": ( + 711, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFEE37, + 0xFFFFFFFFFFFFFFFFFFFFFFFE26F2FC170F69466A74DEFD8D, + 0, + 3, + ( + 0xDB4FF10EC057E9AE26B07D0280B7F4341DA5D1B1EAE06C7D, + 0x9B2F2F6D9C5628A7844163D015BE86344082AA88D95E2F9D + ) + ), + "prime192v1": ( + 409, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFF, + 0xFFFFFFFFFFFFFFFFFFFFFFFF99DEF836146BC9B1B4D22831, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFC, + 0x64210519E59C80E70FA7E9AB72243049FEB8DEECC146B9B1, + ( + 0x188DA80EB03090F67CBF20EB43A18800F4FF0AFD82FF1012, + 0x07192B95FFC8DA78631011ED6B24CDD573F977A11E794811 + ) + ), + "secp224k1": ( + 712, + 0x00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFE56D, + 0x010000000000000000000000000001DCE8D2EC6184CAF0A971769FB1F7, + 0, + 5, + ( + 0xA1455B334DF099DF30FC28A169A467E9E47075A90F7E650EB6B7A45C, + 0x7E089FED7FBA344282CAFBD6F7E319F7C0B0BD59E2CA4BDB556D61A5 + ) + ), + "secp224r1": ( + 713, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF000000000000000000000001, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFF16A2E0B8F03E13DD29455C5C2A3D, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFE, + 0xB4050A850C04B3ABF54132565044B0B7D7BFD8BA270B39432355FFB4, + ( + 0xB70E0CBD6BB4BF7F321390B94A03C1D356C21122343280D6115C1D21, + 0xBD376388B5F723FB4C22DFE6CD4375A05A07476444D5819985007E34 + ) + ), + "secp256k1": ( + 714, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141, + 0, + 7, + ( + 0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798, + 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8 + ) + ), + "prime256v1": ( + 715, + 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF, + 0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551, + 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC, + 0x5AC635D8AA3A93E7B3EBBD55769886BC651D06B0CC53B0F63BCE3C3E27D2604B, + ( + 0x6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296, + 0x4FE342E2FE1A7F9B8EE7EB4A7C0F9E162BCE33576B315ECECBB6406837BF51F5 + ) + ), + "secp384r1": ( + 716, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFF0000000000000000FFFFFFFF, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC7634D81F4372DDF581A0DB248B0A77AECEC196ACCC52973, + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFF0000000000000000FFFFFFFC, + 0xB3312FA7E23EE7E4988E056BE3F82D19181D9C6EFE8141120314088F5013875AC656398D8A2ED19D2A85C8EDD3EC2AEF, + ( + 0xAA87CA22BE8B05378EB1C71EF320AD746E1D3B628BA79B9859F741E082542A385502F25DBF55296C3A545E3872760AB7, + 0x3617DE4A96262C6F5D9E98BF9292DC29F8F41DBD289A147CE9DA3113B5F0B8C00A60B1CE1D7E819D7A431D7C90EA0E5F + ) + ), + "secp521r1": ( + 717, + 0x01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF, + 0x01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA51868783BF2F966B7FCC0148F709A5D03BB5C9B8899C47AEBB6FB71E91386409, + 0x01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC, + 0x0051953EB9618E1C9A1F929A21A0B68540EEA2DA725B99B315F3B8B489918EF109E156193951EC7E937B1652C0BD3BB1BF073573DF883D2C34F1EF451FD46B503F00, + ( + 0x00C6858E06B70404E9CD9E3ECB662395B4429C648139053FB521F828AF606B4D3DBAA14B5E77EFE75928FE1DC127A2FFA8DE3348B3C1856A429BF97E7E31C2E5BD66, + 0x011839296A789A3BC0045C8A5FB42C7D1BD998F54449579B446817AFBD17273E662C97EE72995EF42640C550B9013FAD0761353C7086A272C24088BE94769FD16650 + ) + ) + } + # pylint: enable=line-too-long + + def __init__(self, backend, aes): + self._backend = backend + self._aes = aes + + + def get_curve(self, name): + if name not in self.CURVES: + raise ValueError("Unknown curve {}".format(name)) + nid, p, n, a, b, g = self.CURVES[name] + return EllipticCurve(self._backend(p, n, a, b, g), self._aes, nid) + + + def get_backend(self): + return self._backend.get_backend() + + +class EllipticCurve: + def __init__(self, backend, aes, nid): + self._backend = backend + self._aes = aes + self.nid = nid + + + def _encode_public_key(self, x, y, is_compressed=True, raw=True): + if raw: + if is_compressed: + return bytes([0x02 + (y[-1] % 2)]) + x + else: + return bytes([0x04]) + x + y + else: + return struct.pack("!HH", self.nid, len(x)) + x + struct.pack("!H", len(y)) + y + + + def _decode_public_key(self, public_key, partial=False): + if not public_key: + raise ValueError("No public key") + + if public_key[0] == 0x04: + # Uncompressed + expected_length = 1 + 2 * self._backend.public_key_length + if partial: + if len(public_key) < expected_length: + raise ValueError("Invalid uncompressed public key length") + else: + if len(public_key) != expected_length: + raise ValueError("Invalid uncompressed public key length") + x = public_key[1:1 + self._backend.public_key_length] + y = public_key[1 + self._backend.public_key_length:expected_length] + if partial: + return (x, y), expected_length + else: + return x, y + elif public_key[0] in (0x02, 0x03): + # Compressed + expected_length = 1 + self._backend.public_key_length + if partial: + if len(public_key) < expected_length: + raise ValueError("Invalid compressed public key length") + else: + if len(public_key) != expected_length: + raise ValueError("Invalid compressed public key length") + + x, y = self._backend.decompress_point(public_key[:expected_length]) + # Sanity check + if x != public_key[1:expected_length]: + raise ValueError("Incorrect compressed public key") + if partial: + return (x, y), expected_length + else: + return x, y + else: + raise ValueError("Invalid public key prefix") + + + def _decode_public_key_openssl(self, public_key, partial=False): + if not public_key: + raise ValueError("No public key") + + i = 0 + + nid, = struct.unpack("!H", public_key[i:i + 2]) + i += 2 + if nid != self.nid: + raise ValueError("Wrong curve") + + xlen, = struct.unpack("!H", public_key[i:i + 2]) + i += 2 + if len(public_key) - i < xlen: + raise ValueError("Too short public key") + x = public_key[i:i + xlen] + i += xlen + + ylen, = struct.unpack("!H", public_key[i:i + 2]) + i += 2 + if len(public_key) - i < ylen: + raise ValueError("Too short public key") + y = public_key[i:i + ylen] + i += ylen + + if partial: + return (x, y), i + else: + if i < len(public_key): + raise ValueError("Too long public key") + return x, y + + + def new_private_key(self, is_compressed=False): + return self._backend.new_private_key() + (b"\x01" if is_compressed else b"") + + + def private_to_public(self, private_key): + if len(private_key) == self._backend.public_key_length: + is_compressed = False + elif len(private_key) == self._backend.public_key_length + 1 and private_key[-1] == 1: + is_compressed = True + private_key = private_key[:-1] + else: + raise ValueError("Private key has invalid length") + x, y = self._backend.private_to_public(private_key) + return self._encode_public_key(x, y, is_compressed=is_compressed) + + + def private_to_wif(self, private_key): + return base58.b58encode_check(b"\x80" + private_key) + + + def wif_to_private(self, wif): + dec = base58.b58decode_check(wif) + if dec[0] != 0x80: + raise ValueError("Invalid network (expected mainnet)") + return dec[1:] + + + def public_to_address(self, public_key): + h = hashlib.sha256(public_key).digest() + hash160 = ripemd160(h).digest() + return base58.b58encode_check(b"\x00" + hash160) + + + def private_to_address(self, private_key): + # Kinda useless but left for quick migration from pybitcointools + return self.public_to_address(self.private_to_public(private_key)) + + + def derive(self, private_key, public_key): + if len(private_key) == self._backend.public_key_length + 1 and private_key[-1] == 1: + private_key = private_key[:-1] + if len(private_key) != self._backend.public_key_length: + raise ValueError("Private key has invalid length") + if not isinstance(public_key, tuple): + public_key = self._decode_public_key(public_key) + return self._backend.ecdh(private_key, public_key) + + + def _digest(self, data, hash): + if hash is None: + return data + elif callable(hash): + return hash(data) + elif hash == "sha1": + return hashlib.sha1(data).digest() + elif hash == "sha256": + return hashlib.sha256(data).digest() + elif hash == "sha512": + return hashlib.sha512(data).digest() + else: + raise ValueError("Unknown hash/derivation method") + + + # High-level functions + def encrypt(self, data, public_key, algo="aes-256-cbc", derivation="sha256", mac="hmac-sha256", return_aes_key=False): + # Generate ephemeral private key + private_key = self.new_private_key() + + # Derive key + ecdh = self.derive(private_key, public_key) + key = self._digest(ecdh, derivation) + k_enc_len = self._aes.get_algo_key_length(algo) + if len(key) < k_enc_len: + raise ValueError("Too short digest") + k_enc, k_mac = key[:k_enc_len], key[k_enc_len:] + + # Encrypt + ciphertext, iv = self._aes.encrypt(data, k_enc, algo=algo) + ephem_public_key = self.private_to_public(private_key) + ephem_public_key = self._decode_public_key(ephem_public_key) + ephem_public_key = self._encode_public_key(*ephem_public_key, raw=False) + ciphertext = iv + ephem_public_key + ciphertext + + # Add MAC tag + if callable(mac): + tag = mac(k_mac, ciphertext) + elif mac == "hmac-sha256": + h = hmac.new(k_mac, digestmod="sha256") + h.update(ciphertext) + tag = h.digest() + elif mac == "hmac-sha512": + h = hmac.new(k_mac, digestmod="sha512") + h.update(ciphertext) + tag = h.digest() + elif mac is None: + tag = b"" + else: + raise ValueError("Unsupported MAC") + + if return_aes_key: + return ciphertext + tag, k_enc + else: + return ciphertext + tag + + + def decrypt(self, ciphertext, private_key, algo="aes-256-cbc", derivation="sha256", mac="hmac-sha256"): + # Get MAC tag + if callable(mac): + tag_length = mac.digest_size + elif mac == "hmac-sha256": + tag_length = hmac.new(b"", digestmod="sha256").digest_size + elif mac == "hmac-sha512": + tag_length = hmac.new(b"", digestmod="sha512").digest_size + elif mac is None: + tag_length = 0 + else: + raise ValueError("Unsupported MAC") + + if len(ciphertext) < tag_length: + raise ValueError("Ciphertext is too small to contain MAC tag") + if tag_length == 0: + tag = b"" + else: + ciphertext, tag = ciphertext[:-tag_length], ciphertext[-tag_length:] + + orig_ciphertext = ciphertext + + if len(ciphertext) < 16: + raise ValueError("Ciphertext is too small to contain IV") + iv, ciphertext = ciphertext[:16], ciphertext[16:] + + public_key, pos = self._decode_public_key_openssl(ciphertext, partial=True) + ciphertext = ciphertext[pos:] + + # Derive key + ecdh = self.derive(private_key, public_key) + key = self._digest(ecdh, derivation) + k_enc_len = self._aes.get_algo_key_length(algo) + if len(key) < k_enc_len: + raise ValueError("Too short digest") + k_enc, k_mac = key[:k_enc_len], key[k_enc_len:] + + # Verify MAC tag + if callable(mac): + expected_tag = mac(k_mac, orig_ciphertext) + elif mac == "hmac-sha256": + h = hmac.new(k_mac, digestmod="sha256") + h.update(orig_ciphertext) + expected_tag = h.digest() + elif mac == "hmac-sha512": + h = hmac.new(k_mac, digestmod="sha512") + h.update(orig_ciphertext) + expected_tag = h.digest() + elif mac is None: + expected_tag = b"" + + if not hmac.compare_digest(tag, expected_tag): + raise ValueError("Invalid MAC tag") + + return self._aes.decrypt(ciphertext, iv, k_enc, algo=algo) + + + def sign(self, data, private_key, hash="sha256", recoverable=False, entropy=None): + if len(private_key) == self._backend.public_key_length: + is_compressed = False + elif len(private_key) == self._backend.public_key_length + 1 and private_key[-1] == 1: + is_compressed = True + private_key = private_key[:-1] + else: + raise ValueError("Private key has invalid length") + + data = self._digest(data, hash) + if not entropy: + v = b"\x01" * len(data) + k = b"\x00" * len(data) + k = hmac.new(k, v + b"\x00" + private_key + data, "sha256").digest() + v = hmac.new(k, v, "sha256").digest() + k = hmac.new(k, v + b"\x01" + private_key + data, "sha256").digest() + v = hmac.new(k, v, "sha256").digest() + entropy = hmac.new(k, v, "sha256").digest() + return self._backend.sign(data, private_key, recoverable, is_compressed, entropy=entropy) + + + def recover(self, signature, data, hash="sha256"): + # Sanity check: is this signature recoverable? + if len(signature) != 1 + 2 * self._backend.public_key_length: + raise ValueError("Cannot recover an unrecoverable signature") + x, y = self._backend.recover(signature, self._digest(data, hash)) + is_compressed = signature[0] >= 31 + return self._encode_public_key(x, y, is_compressed=is_compressed) + + + def verify(self, signature, data, public_key, hash="sha256"): + if len(signature) == 1 + 2 * self._backend.public_key_length: + # Recoverable signature + signature = signature[1:] + if len(signature) != 2 * self._backend.public_key_length: + raise ValueError("Invalid signature format") + if not isinstance(public_key, tuple): + public_key = self._decode_public_key(public_key) + return self._backend.verify(signature, self._digest(data, hash), public_key) + + + def derive_child(self, seed, child): + # Based on BIP32 + if not 0 <= child < 2 ** 31: + raise ValueError("Invalid child index") + return self._backend.derive_child(seed, child) diff --git a/src/lib/sslcrypto/_ripemd.py b/src/lib/sslcrypto/_ripemd.py new file mode 100644 index 00000000..89377cc2 --- /dev/null +++ b/src/lib/sslcrypto/_ripemd.py @@ -0,0 +1,375 @@ +# Copyright (c) 2001 Markus Friedl. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: skip-file + +import sys + +digest_size = 20 +digestsize = 20 + +class RIPEMD160: + """ + Return a new RIPEMD160 object. An optional string argument + may be provided; if present, this string will be automatically + hashed. + """ + + def __init__(self, arg=None): + self.ctx = RMDContext() + if arg: + self.update(arg) + self.dig = None + + def update(self, arg): + RMD160Update(self.ctx, arg, len(arg)) + self.dig = None + + def digest(self): + if self.dig: + return self.dig + ctx = self.ctx.copy() + self.dig = RMD160Final(self.ctx) + self.ctx = ctx + return self.dig + + def hexdigest(self): + dig = self.digest() + hex_digest = "" + for d in dig: + hex_digest += "%02x" % d + return hex_digest + + def copy(self): + import copy + return copy.deepcopy(self) + + + +def new(arg=None): + """ + Return a new RIPEMD160 object. An optional string argument + may be provided; if present, this string will be automatically + hashed. + """ + return RIPEMD160(arg) + + + +# +# Private. +# + +class RMDContext: + def __init__(self): + self.state = [0x67452301, 0xEFCDAB89, 0x98BADCFE, + 0x10325476, 0xC3D2E1F0] # uint32 + self.count = 0 # uint64 + self.buffer = [0] * 64 # uchar + def copy(self): + ctx = RMDContext() + ctx.state = self.state[:] + ctx.count = self.count + ctx.buffer = self.buffer[:] + return ctx + +K0 = 0x00000000 +K1 = 0x5A827999 +K2 = 0x6ED9EBA1 +K3 = 0x8F1BBCDC +K4 = 0xA953FD4E + +KK0 = 0x50A28BE6 +KK1 = 0x5C4DD124 +KK2 = 0x6D703EF3 +KK3 = 0x7A6D76E9 +KK4 = 0x00000000 + +def ROL(n, x): + return ((x << n) & 0xffffffff) | (x >> (32 - n)) + +def F0(x, y, z): + return x ^ y ^ z + +def F1(x, y, z): + return (x & y) | (((~x) % 0x100000000) & z) + +def F2(x, y, z): + return (x | ((~y) % 0x100000000)) ^ z + +def F3(x, y, z): + return (x & z) | (((~z) % 0x100000000) & y) + +def F4(x, y, z): + return x ^ (y | ((~z) % 0x100000000)) + +def R(a, b, c, d, e, Fj, Kj, sj, rj, X): + a = ROL(sj, (a + Fj(b, c, d) + X[rj] + Kj) % 0x100000000) + e + c = ROL(10, c) + return a % 0x100000000, c + +PADDING = [0x80] + [0] * 63 + +import sys +import struct + +def RMD160Transform(state, block): # uint32 state[5], uchar block[64] + x = [0] * 16 + if sys.byteorder == "little": + x = struct.unpack("<16L", bytes(block[0:64])) + else: + raise ValueError("Big-endian platforms are not supported") + a = state[0] + b = state[1] + c = state[2] + d = state[3] + e = state[4] + + # Round 1 + a, c = R(a, b, c, d, e, F0, K0, 11, 0, x) + e, b = R(e, a, b, c, d, F0, K0, 14, 1, x) + d, a = R(d, e, a, b, c, F0, K0, 15, 2, x) + c, e = R(c, d, e, a, b, F0, K0, 12, 3, x) + b, d = R(b, c, d, e, a, F0, K0, 5, 4, x) + a, c = R(a, b, c, d, e, F0, K0, 8, 5, x) + e, b = R(e, a, b, c, d, F0, K0, 7, 6, x) + d, a = R(d, e, a, b, c, F0, K0, 9, 7, x) + c, e = R(c, d, e, a, b, F0, K0, 11, 8, x) + b, d = R(b, c, d, e, a, F0, K0, 13, 9, x) + a, c = R(a, b, c, d, e, F0, K0, 14, 10, x) + e, b = R(e, a, b, c, d, F0, K0, 15, 11, x) + d, a = R(d, e, a, b, c, F0, K0, 6, 12, x) + c, e = R(c, d, e, a, b, F0, K0, 7, 13, x) + b, d = R(b, c, d, e, a, F0, K0, 9, 14, x) + a, c = R(a, b, c, d, e, F0, K0, 8, 15, x) # #15 + # Round 2 + e, b = R(e, a, b, c, d, F1, K1, 7, 7, x) + d, a = R(d, e, a, b, c, F1, K1, 6, 4, x) + c, e = R(c, d, e, a, b, F1, K1, 8, 13, x) + b, d = R(b, c, d, e, a, F1, K1, 13, 1, x) + a, c = R(a, b, c, d, e, F1, K1, 11, 10, x) + e, b = R(e, a, b, c, d, F1, K1, 9, 6, x) + d, a = R(d, e, a, b, c, F1, K1, 7, 15, x) + c, e = R(c, d, e, a, b, F1, K1, 15, 3, x) + b, d = R(b, c, d, e, a, F1, K1, 7, 12, x) + a, c = R(a, b, c, d, e, F1, K1, 12, 0, x) + e, b = R(e, a, b, c, d, F1, K1, 15, 9, x) + d, a = R(d, e, a, b, c, F1, K1, 9, 5, x) + c, e = R(c, d, e, a, b, F1, K1, 11, 2, x) + b, d = R(b, c, d, e, a, F1, K1, 7, 14, x) + a, c = R(a, b, c, d, e, F1, K1, 13, 11, x) + e, b = R(e, a, b, c, d, F1, K1, 12, 8, x) # #31 + # Round 3 + d, a = R(d, e, a, b, c, F2, K2, 11, 3, x) + c, e = R(c, d, e, a, b, F2, K2, 13, 10, x) + b, d = R(b, c, d, e, a, F2, K2, 6, 14, x) + a, c = R(a, b, c, d, e, F2, K2, 7, 4, x) + e, b = R(e, a, b, c, d, F2, K2, 14, 9, x) + d, a = R(d, e, a, b, c, F2, K2, 9, 15, x) + c, e = R(c, d, e, a, b, F2, K2, 13, 8, x) + b, d = R(b, c, d, e, a, F2, K2, 15, 1, x) + a, c = R(a, b, c, d, e, F2, K2, 14, 2, x) + e, b = R(e, a, b, c, d, F2, K2, 8, 7, x) + d, a = R(d, e, a, b, c, F2, K2, 13, 0, x) + c, e = R(c, d, e, a, b, F2, K2, 6, 6, x) + b, d = R(b, c, d, e, a, F2, K2, 5, 13, x) + a, c = R(a, b, c, d, e, F2, K2, 12, 11, x) + e, b = R(e, a, b, c, d, F2, K2, 7, 5, x) + d, a = R(d, e, a, b, c, F2, K2, 5, 12, x) # #47 + # Round 4 + c, e = R(c, d, e, a, b, F3, K3, 11, 1, x) + b, d = R(b, c, d, e, a, F3, K3, 12, 9, x) + a, c = R(a, b, c, d, e, F3, K3, 14, 11, x) + e, b = R(e, a, b, c, d, F3, K3, 15, 10, x) + d, a = R(d, e, a, b, c, F3, K3, 14, 0, x) + c, e = R(c, d, e, a, b, F3, K3, 15, 8, x) + b, d = R(b, c, d, e, a, F3, K3, 9, 12, x) + a, c = R(a, b, c, d, e, F3, K3, 8, 4, x) + e, b = R(e, a, b, c, d, F3, K3, 9, 13, x) + d, a = R(d, e, a, b, c, F3, K3, 14, 3, x) + c, e = R(c, d, e, a, b, F3, K3, 5, 7, x) + b, d = R(b, c, d, e, a, F3, K3, 6, 15, x) + a, c = R(a, b, c, d, e, F3, K3, 8, 14, x) + e, b = R(e, a, b, c, d, F3, K3, 6, 5, x) + d, a = R(d, e, a, b, c, F3, K3, 5, 6, x) + c, e = R(c, d, e, a, b, F3, K3, 12, 2, x) # #63 + # Round 5 + b, d = R(b, c, d, e, a, F4, K4, 9, 4, x) + a, c = R(a, b, c, d, e, F4, K4, 15, 0, x) + e, b = R(e, a, b, c, d, F4, K4, 5, 5, x) + d, a = R(d, e, a, b, c, F4, K4, 11, 9, x) + c, e = R(c, d, e, a, b, F4, K4, 6, 7, x) + b, d = R(b, c, d, e, a, F4, K4, 8, 12, x) + a, c = R(a, b, c, d, e, F4, K4, 13, 2, x) + e, b = R(e, a, b, c, d, F4, K4, 12, 10, x) + d, a = R(d, e, a, b, c, F4, K4, 5, 14, x) + c, e = R(c, d, e, a, b, F4, K4, 12, 1, x) + b, d = R(b, c, d, e, a, F4, K4, 13, 3, x) + a, c = R(a, b, c, d, e, F4, K4, 14, 8, x) + e, b = R(e, a, b, c, d, F4, K4, 11, 11, x) + d, a = R(d, e, a, b, c, F4, K4, 8, 6, x) + c, e = R(c, d, e, a, b, F4, K4, 5, 15, x) + b, d = R(b, c, d, e, a, F4, K4, 6, 13, x) # #79 + + aa = a + bb = b + cc = c + dd = d + ee = e + + a = state[0] + b = state[1] + c = state[2] + d = state[3] + e = state[4] + + # Parallel round 1 + a, c = R(a, b, c, d, e, F4, KK0, 8, 5, x) + e, b = R(e, a, b, c, d, F4, KK0, 9, 14, x) + d, a = R(d, e, a, b, c, F4, KK0, 9, 7, x) + c, e = R(c, d, e, a, b, F4, KK0, 11, 0, x) + b, d = R(b, c, d, e, a, F4, KK0, 13, 9, x) + a, c = R(a, b, c, d, e, F4, KK0, 15, 2, x) + e, b = R(e, a, b, c, d, F4, KK0, 15, 11, x) + d, a = R(d, e, a, b, c, F4, KK0, 5, 4, x) + c, e = R(c, d, e, a, b, F4, KK0, 7, 13, x) + b, d = R(b, c, d, e, a, F4, KK0, 7, 6, x) + a, c = R(a, b, c, d, e, F4, KK0, 8, 15, x) + e, b = R(e, a, b, c, d, F4, KK0, 11, 8, x) + d, a = R(d, e, a, b, c, F4, KK0, 14, 1, x) + c, e = R(c, d, e, a, b, F4, KK0, 14, 10, x) + b, d = R(b, c, d, e, a, F4, KK0, 12, 3, x) + a, c = R(a, b, c, d, e, F4, KK0, 6, 12, x) # #15 + # Parallel round 2 + e, b = R(e, a, b, c, d, F3, KK1, 9, 6, x) + d, a = R(d, e, a, b, c, F3, KK1, 13, 11, x) + c, e = R(c, d, e, a, b, F3, KK1, 15, 3, x) + b, d = R(b, c, d, e, a, F3, KK1, 7, 7, x) + a, c = R(a, b, c, d, e, F3, KK1, 12, 0, x) + e, b = R(e, a, b, c, d, F3, KK1, 8, 13, x) + d, a = R(d, e, a, b, c, F3, KK1, 9, 5, x) + c, e = R(c, d, e, a, b, F3, KK1, 11, 10, x) + b, d = R(b, c, d, e, a, F3, KK1, 7, 14, x) + a, c = R(a, b, c, d, e, F3, KK1, 7, 15, x) + e, b = R(e, a, b, c, d, F3, KK1, 12, 8, x) + d, a = R(d, e, a, b, c, F3, KK1, 7, 12, x) + c, e = R(c, d, e, a, b, F3, KK1, 6, 4, x) + b, d = R(b, c, d, e, a, F3, KK1, 15, 9, x) + a, c = R(a, b, c, d, e, F3, KK1, 13, 1, x) + e, b = R(e, a, b, c, d, F3, KK1, 11, 2, x) # #31 + # Parallel round 3 + d, a = R(d, e, a, b, c, F2, KK2, 9, 15, x) + c, e = R(c, d, e, a, b, F2, KK2, 7, 5, x) + b, d = R(b, c, d, e, a, F2, KK2, 15, 1, x) + a, c = R(a, b, c, d, e, F2, KK2, 11, 3, x) + e, b = R(e, a, b, c, d, F2, KK2, 8, 7, x) + d, a = R(d, e, a, b, c, F2, KK2, 6, 14, x) + c, e = R(c, d, e, a, b, F2, KK2, 6, 6, x) + b, d = R(b, c, d, e, a, F2, KK2, 14, 9, x) + a, c = R(a, b, c, d, e, F2, KK2, 12, 11, x) + e, b = R(e, a, b, c, d, F2, KK2, 13, 8, x) + d, a = R(d, e, a, b, c, F2, KK2, 5, 12, x) + c, e = R(c, d, e, a, b, F2, KK2, 14, 2, x) + b, d = R(b, c, d, e, a, F2, KK2, 13, 10, x) + a, c = R(a, b, c, d, e, F2, KK2, 13, 0, x) + e, b = R(e, a, b, c, d, F2, KK2, 7, 4, x) + d, a = R(d, e, a, b, c, F2, KK2, 5, 13, x) # #47 + # Parallel round 4 + c, e = R(c, d, e, a, b, F1, KK3, 15, 8, x) + b, d = R(b, c, d, e, a, F1, KK3, 5, 6, x) + a, c = R(a, b, c, d, e, F1, KK3, 8, 4, x) + e, b = R(e, a, b, c, d, F1, KK3, 11, 1, x) + d, a = R(d, e, a, b, c, F1, KK3, 14, 3, x) + c, e = R(c, d, e, a, b, F1, KK3, 14, 11, x) + b, d = R(b, c, d, e, a, F1, KK3, 6, 15, x) + a, c = R(a, b, c, d, e, F1, KK3, 14, 0, x) + e, b = R(e, a, b, c, d, F1, KK3, 6, 5, x) + d, a = R(d, e, a, b, c, F1, KK3, 9, 12, x) + c, e = R(c, d, e, a, b, F1, KK3, 12, 2, x) + b, d = R(b, c, d, e, a, F1, KK3, 9, 13, x) + a, c = R(a, b, c, d, e, F1, KK3, 12, 9, x) + e, b = R(e, a, b, c, d, F1, KK3, 5, 7, x) + d, a = R(d, e, a, b, c, F1, KK3, 15, 10, x) + c, e = R(c, d, e, a, b, F1, KK3, 8, 14, x) # #63 + # Parallel round 5 + b, d = R(b, c, d, e, a, F0, KK4, 8, 12, x) + a, c = R(a, b, c, d, e, F0, KK4, 5, 15, x) + e, b = R(e, a, b, c, d, F0, KK4, 12, 10, x) + d, a = R(d, e, a, b, c, F0, KK4, 9, 4, x) + c, e = R(c, d, e, a, b, F0, KK4, 12, 1, x) + b, d = R(b, c, d, e, a, F0, KK4, 5, 5, x) + a, c = R(a, b, c, d, e, F0, KK4, 14, 8, x) + e, b = R(e, a, b, c, d, F0, KK4, 6, 7, x) + d, a = R(d, e, a, b, c, F0, KK4, 8, 6, x) + c, e = R(c, d, e, a, b, F0, KK4, 13, 2, x) + b, d = R(b, c, d, e, a, F0, KK4, 6, 13, x) + a, c = R(a, b, c, d, e, F0, KK4, 5, 14, x) + e, b = R(e, a, b, c, d, F0, KK4, 15, 0, x) + d, a = R(d, e, a, b, c, F0, KK4, 13, 3, x) + c, e = R(c, d, e, a, b, F0, KK4, 11, 9, x) + b, d = R(b, c, d, e, a, F0, KK4, 11, 11, x) # #79 + + t = (state[1] + cc + d) % 0x100000000 + state[1] = (state[2] + dd + e) % 0x100000000 + state[2] = (state[3] + ee + a) % 0x100000000 + state[3] = (state[4] + aa + b) % 0x100000000 + state[4] = (state[0] + bb + c) % 0x100000000 + state[0] = t % 0x100000000 + + +def RMD160Update(ctx, inp, inplen): + if type(inp) == str: + inp = [ord(i)&0xff for i in inp] + + have = int((ctx.count // 8) % 64) + inplen = int(inplen) + need = 64 - have + ctx.count += 8 * inplen + off = 0 + if inplen >= need: + if have: + for i in range(need): + ctx.buffer[have + i] = inp[i] + RMD160Transform(ctx.state, ctx.buffer) + off = need + have = 0 + while off + 64 <= inplen: + RMD160Transform(ctx.state, inp[off:]) #<--- + off += 64 + if off < inplen: + # memcpy(ctx->buffer + have, input+off, len-off) + for i in range(inplen - off): + ctx.buffer[have + i] = inp[off + i] + +def RMD160Final(ctx): + size = struct.pack("= self.n: + return self.jacobian_multiply(a, n % self.n, secret) + half = self.jacobian_multiply(a, n // 2, secret) + half_sq = self.jacobian_double(half) + if secret: + # A constant-time implementation + half_sq_a = self.jacobian_add(half_sq, a) + if n % 2 == 0: + result = half_sq + if n % 2 == 1: + result = half_sq_a + return result + else: + if n % 2 == 0: + return half_sq + return self.jacobian_add(half_sq, a) + + + def jacobian_shamir(self, a, n, b, m): + ab = self.jacobian_add(a, b) + if n < 0 or n >= self.n: + n %= self.n + if m < 0 or m >= self.n: + m %= self.n + res = 0, 0, 1 # point on infinity + for i in range(self.n_length - 1, -1, -1): + res = self.jacobian_double(res) + has_n = n & (1 << i) + has_m = m & (1 << i) + if has_n: + if has_m == 0: + res = self.jacobian_add(res, a) + if has_m != 0: + res = self.jacobian_add(res, ab) + else: + if has_m == 0: + res = self.jacobian_add(res, (0, 0, 1)) # Try not to leak + if has_m != 0: + res = self.jacobian_add(res, b) + return res + + + def fast_multiply(self, a, n, secret=False): + return self.from_jacobian(self.jacobian_multiply(self.to_jacobian(a), n, secret)) + + + def fast_add(self, a, b): + return self.from_jacobian(self.jacobian_add(self.to_jacobian(a), self.to_jacobian(b))) + + + def fast_shamir(self, a, n, b, m): + return self.from_jacobian(self.jacobian_shamir(self.to_jacobian(a), n, self.to_jacobian(b), m)) + + + def is_on_curve(self, a): + x, y = a + # Simple arithmetic check + if (pow(x, 3, self.p) + self.a * x + self.b) % self.p != y * y % self.p: + return False + # nP = point-at-infinity + return self.isinf(self.jacobian_multiply(self.to_jacobian(a), self.n)) diff --git a/src/lib/sslcrypto/fallback/_util.py b/src/lib/sslcrypto/fallback/_util.py new file mode 100644 index 00000000..2236ebee --- /dev/null +++ b/src/lib/sslcrypto/fallback/_util.py @@ -0,0 +1,79 @@ +def int_to_bytes(raw, length): + data = [] + for _ in range(length): + data.append(raw % 256) + raw //= 256 + return bytes(data[::-1]) + + +def bytes_to_int(data): + raw = 0 + for byte in data: + raw = raw * 256 + byte + return raw + + +def legendre(a, p): + res = pow(a, (p - 1) // 2, p) + if res == p - 1: + return -1 + else: + return res + + +def inverse(a, n): + if a == 0: + return 0 + lm, hm = 1, 0 + low, high = a % n, n + while low > 1: + r = high // low + nm, new = hm - lm * r, high - low * r + lm, low, hm, high = nm, new, lm, low + return lm % n + + +def square_root_mod_prime(n, p): + if n == 0: + return 0 + if p == 2: + return n # We should never get here but it might be useful + if legendre(n, p) != 1: + raise ValueError("No square root") + # Optimizations + if p % 4 == 3: + return pow(n, (p + 1) // 4, p) + # 1. By factoring out powers of 2, find Q and S such that p - 1 = + # Q * 2 ** S with Q odd + q = p - 1 + s = 0 + while q % 2 == 0: + q //= 2 + s += 1 + # 2. Search for z in Z/pZ which is a quadratic non-residue + z = 1 + while legendre(z, p) != -1: + z += 1 + m, c, t, r = s, pow(z, q, p), pow(n, q, p), pow(n, (q + 1) // 2, p) + while True: + if t == 0: + return 0 + elif t == 1: + return r + # Use repeated squaring to find the least i, 0 < i < M, such + # that t ** (2 ** i) = 1 + t_sq = t + i = 0 + for i in range(1, m): + t_sq = t_sq * t_sq % p + if t_sq == 1: + break + else: + raise ValueError("Should never get here") + # Let b = c ** (2 ** (m - i - 1)) + b = pow(c, 2 ** (m - i - 1), p) + m = i + c = b * b % p + t = t * b * b % p + r = r * b % p + return r diff --git a/src/lib/sslcrypto/fallback/aes.py b/src/lib/sslcrypto/fallback/aes.py new file mode 100644 index 00000000..e168bf34 --- /dev/null +++ b/src/lib/sslcrypto/fallback/aes.py @@ -0,0 +1,101 @@ +import os +import pyaes +from .._aes import AES + + +__all__ = ["aes"] + +class AESBackend: + def _get_algo_cipher_type(self, algo): + if not algo.startswith("aes-") or algo.count("-") != 2: + raise ValueError("Unknown cipher algorithm {}".format(algo)) + key_length, cipher_type = algo[4:].split("-") + if key_length not in ("128", "192", "256"): + raise ValueError("Unknown cipher algorithm {}".format(algo)) + if cipher_type not in ("cbc", "ctr", "cfb", "ofb"): + raise ValueError("Unknown cipher algorithm {}".format(algo)) + return cipher_type + + + def is_algo_supported(self, algo): + try: + self._get_algo_cipher_type(algo) + return True + except ValueError: + return False + + + def random(self, length): + return os.urandom(length) + + + def encrypt(self, data, key, algo="aes-256-cbc"): + cipher_type = self._get_algo_cipher_type(algo) + + # Generate random IV + iv = os.urandom(16) + + if cipher_type == "cbc": + cipher = pyaes.AESModeOfOperationCBC(key, iv=iv) + elif cipher_type == "ctr": + # The IV is actually a counter, not an IV but it does almost the + # same. Notice: pyaes always uses 1 as initial counter! Make sure + # not to call pyaes directly. + + # We kinda do two conversions here: from byte array to int here, and + # from int to byte array in pyaes internals. It's possible to fix that + # but I didn't notice any performance changes so I'm keeping clean code. + iv_int = 0 + for byte in iv: + iv_int = (iv_int * 256) + byte + counter = pyaes.Counter(iv_int) + cipher = pyaes.AESModeOfOperationCTR(key, counter=counter) + elif cipher_type == "cfb": + # Change segment size from default 8 bytes to 16 bytes for OpenSSL + # compatibility + cipher = pyaes.AESModeOfOperationCFB(key, iv, segment_size=16) + elif cipher_type == "ofb": + cipher = pyaes.AESModeOfOperationOFB(key, iv) + + encrypter = pyaes.Encrypter(cipher) + ciphertext = encrypter.feed(data) + ciphertext += encrypter.feed() + return ciphertext, iv + + + def decrypt(self, ciphertext, iv, key, algo="aes-256-cbc"): + cipher_type = self._get_algo_cipher_type(algo) + + if cipher_type == "cbc": + cipher = pyaes.AESModeOfOperationCBC(key, iv=iv) + elif cipher_type == "ctr": + # The IV is actually a counter, not an IV but it does almost the + # same. Notice: pyaes always uses 1 as initial counter! Make sure + # not to call pyaes directly. + + # We kinda do two conversions here: from byte array to int here, and + # from int to byte array in pyaes internals. It's possible to fix that + # but I didn't notice any performance changes so I'm keeping clean code. + iv_int = 0 + for byte in iv: + iv_int = (iv_int * 256) + byte + counter = pyaes.Counter(iv_int) + cipher = pyaes.AESModeOfOperationCTR(key, counter=counter) + elif cipher_type == "cfb": + # Change segment size from default 8 bytes to 16 bytes for OpenSSL + # compatibility + cipher = pyaes.AESModeOfOperationCFB(key, iv, segment_size=16) + elif cipher_type == "ofb": + cipher = pyaes.AESModeOfOperationOFB(key, iv) + + decrypter = pyaes.Decrypter(cipher) + data = decrypter.feed(ciphertext) + data += decrypter.feed() + return data + + + def get_backend(self): + return "fallback" + + +aes = AES(AESBackend()) diff --git a/src/lib/sslcrypto/fallback/ecc.py b/src/lib/sslcrypto/fallback/ecc.py new file mode 100644 index 00000000..6ca9a498 --- /dev/null +++ b/src/lib/sslcrypto/fallback/ecc.py @@ -0,0 +1,199 @@ +import hmac +import os +from ._jacobian import JacobianCurve +from .._ecc import ECC +from .aes import aes +from ._util import int_to_bytes, bytes_to_int, inverse, square_root_mod_prime + + +class EllipticCurveBackend: + def __init__(self, p, n, a, b, g): + self.p, self.n, self.a, self.b, self.g = p, n, a, b, g + self.jacobian = JacobianCurve(p, n, a, b, g) + + self.public_key_length = (len(bin(p).replace("0b", "")) + 7) // 8 + self.order_bitlength = len(bin(n).replace("0b", "")) + + + def _int_to_bytes(self, raw, len=None): + return int_to_bytes(raw, len or self.public_key_length) + + + def decompress_point(self, public_key): + # Parse & load data + x = bytes_to_int(public_key[1:]) + # Calculate Y + y_square = (pow(x, 3, self.p) + self.a * x + self.b) % self.p + try: + y = square_root_mod_prime(y_square, self.p) + except Exception: + raise ValueError("Invalid public key") from None + if y % 2 != public_key[0] - 0x02: + y = self.p - y + return self._int_to_bytes(x), self._int_to_bytes(y) + + + def new_private_key(self): + while True: + private_key = os.urandom(self.public_key_length) + if bytes_to_int(private_key) >= self.n: + continue + return private_key + + + def private_to_public(self, private_key): + raw = bytes_to_int(private_key) + x, y = self.jacobian.fast_multiply(self.g, raw) + return self._int_to_bytes(x), self._int_to_bytes(y) + + + def ecdh(self, private_key, public_key): + x, y = public_key + x, y = bytes_to_int(x), bytes_to_int(y) + private_key = bytes_to_int(private_key) + x, _ = self.jacobian.fast_multiply((x, y), private_key, secret=True) + return self._int_to_bytes(x) + + + def _subject_to_int(self, subject): + return bytes_to_int(subject[:(self.order_bitlength + 7) // 8]) + + + def sign(self, subject, raw_private_key, recoverable, is_compressed, entropy): + z = self._subject_to_int(subject) + private_key = bytes_to_int(raw_private_key) + k = bytes_to_int(entropy) + + # Fix k length to prevent Minerva. Increasing multiplier by a + # multiple of order doesn't break anything. This fix was ported + # from python-ecdsa + ks = k + self.n + kt = ks + self.n + ks_len = len(bin(ks).replace("0b", "")) // 8 + kt_len = len(bin(kt).replace("0b", "")) // 8 + if ks_len == kt_len: + k = kt + else: + k = ks + px, py = self.jacobian.fast_multiply(self.g, k, secret=True) + + r = px % self.n + if r == 0: + # Invalid k + raise ValueError("Invalid k") + + s = (inverse(k, self.n) * (z + (private_key * r))) % self.n + if s == 0: + # Invalid k + raise ValueError("Invalid k") + + inverted = False + if s * 2 >= self.n: + s = self.n - s + inverted = True + rs_buf = self._int_to_bytes(r) + self._int_to_bytes(s) + + if recoverable: + recid = (py % 2) ^ inverted + recid += 2 * int(px // self.n) + if is_compressed: + return bytes([31 + recid]) + rs_buf + else: + if recid >= 4: + raise ValueError("Too big recovery ID, use compressed address instead") + return bytes([27 + recid]) + rs_buf + else: + return rs_buf + + + def recover(self, signature, subject): + z = self._subject_to_int(subject) + + recid = signature[0] - 27 if signature[0] < 31 else signature[0] - 31 + r = bytes_to_int(signature[1:self.public_key_length + 1]) + s = bytes_to_int(signature[self.public_key_length + 1:]) + + # Verify bounds + if not 0 <= recid < 2 * (self.p // self.n + 1): + raise ValueError("Invalid recovery ID") + if r >= self.n: + raise ValueError("r is out of bounds") + if s >= self.n: + raise ValueError("s is out of bounds") + + rinv = inverse(r, self.n) + u1 = (-z * rinv) % self.n + u2 = (s * rinv) % self.n + + # Recover R + rx = r + (recid // 2) * self.n + if rx >= self.p: + raise ValueError("Rx is out of bounds") + + # Almost copied from decompress_point + ry_square = (pow(rx, 3, self.p) + self.a * rx + self.b) % self.p + try: + ry = square_root_mod_prime(ry_square, self.p) + except Exception: + raise ValueError("Invalid recovered public key") from None + + # Ensure the point is correct + if ry % 2 != recid % 2: + # Fix Ry sign + ry = self.p - ry + + x, y = self.jacobian.fast_shamir(self.g, u1, (rx, ry), u2) + return self._int_to_bytes(x), self._int_to_bytes(y) + + + def verify(self, signature, subject, public_key): + z = self._subject_to_int(subject) + + r = bytes_to_int(signature[:self.public_key_length]) + s = bytes_to_int(signature[self.public_key_length:]) + + # Verify bounds + if r >= self.n: + raise ValueError("r is out of bounds") + if s >= self.n: + raise ValueError("s is out of bounds") + + public_key = [bytes_to_int(c) for c in public_key] + + # Ensure that the public key is correct + if not self.jacobian.is_on_curve(public_key): + raise ValueError("Public key is not on curve") + + sinv = inverse(s, self.n) + u1 = (z * sinv) % self.n + u2 = (r * sinv) % self.n + + x1, _ = self.jacobian.fast_shamir(self.g, u1, public_key, u2) + if r != x1 % self.n: + raise ValueError("Invalid signature") + + return True + + + def derive_child(self, seed, child): + # Round 1 + h = hmac.new(key=b"Bitcoin seed", msg=seed, digestmod="sha512").digest() + private_key1 = h[:32] + x, y = self.private_to_public(private_key1) + public_key1 = bytes([0x02 + (y[-1] % 2)]) + x + private_key1 = bytes_to_int(private_key1) + + # Round 2 + msg = public_key1 + self._int_to_bytes(child, 4) + h = hmac.new(key=h[32:], msg=msg, digestmod="sha512").digest() + private_key2 = bytes_to_int(h[:32]) + + return self._int_to_bytes((private_key1 + private_key2) % self.n) + + + @classmethod + def get_backend(cls): + return "fallback" + + +ecc = ECC(EllipticCurveBackend, aes) diff --git a/src/lib/sslcrypto/fallback/rsa.py b/src/lib/sslcrypto/fallback/rsa.py new file mode 100644 index 00000000..54b8d2cb --- /dev/null +++ b/src/lib/sslcrypto/fallback/rsa.py @@ -0,0 +1,8 @@ +# pylint: disable=too-few-public-methods + +class RSA: + def get_backend(self): + return "fallback" + + +rsa = RSA() diff --git a/src/lib/sslcrypto/openssl/__init__.py b/src/lib/sslcrypto/openssl/__init__.py new file mode 100644 index 00000000..a32ae692 --- /dev/null +++ b/src/lib/sslcrypto/openssl/__init__.py @@ -0,0 +1,3 @@ +from .aes import aes +from .ecc import ecc +from .rsa import rsa diff --git a/src/lib/sslcrypto/openssl/aes.py b/src/lib/sslcrypto/openssl/aes.py new file mode 100644 index 00000000..c58451d5 --- /dev/null +++ b/src/lib/sslcrypto/openssl/aes.py @@ -0,0 +1,156 @@ +import ctypes +import threading +from .._aes import AES +from ..fallback.aes import aes as fallback_aes +from .library import lib, openssl_backend + + +# Initialize functions +try: + lib.EVP_CIPHER_CTX_new.restype = ctypes.POINTER(ctypes.c_char) +except AttributeError: + pass +lib.EVP_get_cipherbyname.restype = ctypes.POINTER(ctypes.c_char) + + +thread_local = threading.local() + + +class Context: + def __init__(self, ptr, do_free): + self.lib = lib + self.ptr = ptr + self.do_free = do_free + + + def __del__(self): + if self.do_free: + self.lib.EVP_CIPHER_CTX_free(self.ptr) + + +class AESBackend: + ALGOS = ( + "aes-128-cbc", "aes-192-cbc", "aes-256-cbc", + "aes-128-ctr", "aes-192-ctr", "aes-256-ctr", + "aes-128-cfb", "aes-192-cfb", "aes-256-cfb", + "aes-128-ofb", "aes-192-ofb", "aes-256-ofb" + ) + + def __init__(self): + self.is_supported_ctx_new = hasattr(lib, "EVP_CIPHER_CTX_new") + self.is_supported_ctx_reset = hasattr(lib, "EVP_CIPHER_CTX_reset") + + + def _get_ctx(self): + if not hasattr(thread_local, "ctx"): + if self.is_supported_ctx_new: + thread_local.ctx = Context(lib.EVP_CIPHER_CTX_new(), True) + else: + # 1 KiB ought to be enough for everybody. We don't know the real + # size of the context buffer because we are unsure about padding and + # pointer size + thread_local.ctx = Context(ctypes.create_string_buffer(1024), False) + return thread_local.ctx.ptr + + + def get_backend(self): + return openssl_backend + + + def _get_cipher(self, algo): + if algo not in self.ALGOS: + raise ValueError("Unknown cipher algorithm {}".format(algo)) + cipher = lib.EVP_get_cipherbyname(algo.encode()) + if not cipher: + raise ValueError("Unknown cipher algorithm {}".format(algo)) + return cipher + + + def is_algo_supported(self, algo): + try: + self._get_cipher(algo) + return True + except ValueError: + return False + + + def random(self, length): + entropy = ctypes.create_string_buffer(length) + lib.RAND_bytes(entropy, length) + return bytes(entropy) + + + def encrypt(self, data, key, algo="aes-256-cbc"): + # Initialize context + ctx = self._get_ctx() + if not self.is_supported_ctx_new: + lib.EVP_CIPHER_CTX_init(ctx) + try: + lib.EVP_EncryptInit_ex(ctx, self._get_cipher(algo), None, None, None) + + # Generate random IV + iv_length = 16 + iv = self.random(iv_length) + + # Set key and IV + lib.EVP_EncryptInit_ex(ctx, None, None, key, iv) + + # Actually encrypt + block_size = 16 + output = ctypes.create_string_buffer((len(data) // block_size + 1) * block_size) + output_len = ctypes.c_int() + + if not lib.EVP_CipherUpdate(ctx, output, ctypes.byref(output_len), data, len(data)): + raise ValueError("Could not feed cipher with data") + + new_output = ctypes.byref(output, output_len.value) + output_len2 = ctypes.c_int() + if not lib.EVP_CipherFinal_ex(ctx, new_output, ctypes.byref(output_len2)): + raise ValueError("Could not finalize cipher") + + ciphertext = output[:output_len.value + output_len2.value] + return ciphertext, iv + finally: + if self.is_supported_ctx_reset: + lib.EVP_CIPHER_CTX_reset(ctx) + else: + lib.EVP_CIPHER_CTX_cleanup(ctx) + + + def decrypt(self, ciphertext, iv, key, algo="aes-256-cbc"): + # Initialize context + ctx = self._get_ctx() + if not self.is_supported_ctx_new: + lib.EVP_CIPHER_CTX_init(ctx) + try: + lib.EVP_DecryptInit_ex(ctx, self._get_cipher(algo), None, None, None) + + # Make sure IV length is correct + iv_length = 16 + if len(iv) != iv_length: + raise ValueError("Expected IV to be {} bytes, got {} bytes".format(iv_length, len(iv))) + + # Set key and IV + lib.EVP_DecryptInit_ex(ctx, None, None, key, iv) + + # Actually decrypt + output = ctypes.create_string_buffer(len(ciphertext)) + output_len = ctypes.c_int() + + if not lib.EVP_DecryptUpdate(ctx, output, ctypes.byref(output_len), ciphertext, len(ciphertext)): + raise ValueError("Could not feed decipher with ciphertext") + + new_output = ctypes.byref(output, output_len.value) + output_len2 = ctypes.c_int() + if not lib.EVP_DecryptFinal_ex(ctx, new_output, ctypes.byref(output_len2)): + raise ValueError("Could not finalize decipher") + + return output[:output_len.value + output_len2.value] + finally: + if self.is_supported_ctx_reset: + lib.EVP_CIPHER_CTX_reset(ctx) + else: + lib.EVP_CIPHER_CTX_cleanup(ctx) + + +aes = AES(AESBackend(), fallback_aes) diff --git a/src/lib/sslcrypto/openssl/discovery.py b/src/lib/sslcrypto/openssl/discovery.py new file mode 100644 index 00000000..0ebb0299 --- /dev/null +++ b/src/lib/sslcrypto/openssl/discovery.py @@ -0,0 +1,3 @@ +# Can be redefined by user +def discover(): + pass \ No newline at end of file diff --git a/src/lib/sslcrypto/openssl/ecc.py b/src/lib/sslcrypto/openssl/ecc.py new file mode 100644 index 00000000..c667be8a --- /dev/null +++ b/src/lib/sslcrypto/openssl/ecc.py @@ -0,0 +1,583 @@ +import ctypes +import hmac +import threading +from .._ecc import ECC +from .aes import aes +from .library import lib, openssl_backend + + +# Initialize functions +lib.BN_new.restype = ctypes.POINTER(ctypes.c_char) +lib.BN_bin2bn.restype = ctypes.POINTER(ctypes.c_char) +lib.BN_CTX_new.restype = ctypes.POINTER(ctypes.c_char) +lib.EC_GROUP_new_curve_GFp.restype = ctypes.POINTER(ctypes.c_char) +lib.EC_KEY_new.restype = ctypes.POINTER(ctypes.c_char) +lib.EC_POINT_new.restype = ctypes.POINTER(ctypes.c_char) +lib.EC_KEY_get0_private_key.restype = ctypes.POINTER(ctypes.c_char) +lib.EVP_PKEY_new.restype = ctypes.POINTER(ctypes.c_char) +try: + lib.EVP_PKEY_CTX_new.restype = ctypes.POINTER(ctypes.c_char) +except AttributeError: + pass + + +thread_local = threading.local() + + +# This lock is required to keep ECC thread-safe. Old OpenSSL versions (before +# 1.1.0) use global objects so they aren't thread safe. Fortunately we can check +# the code to find out which functions are thread safe. +# +# For example, EC_GROUP_new_curve_GFp checks global error code to initialize +# the group, so if two errors happen at once or two threads read the error code, +# or the codes are read in the wrong order, the group is initialized in a wrong +# way. +# +# EC_KEY_new_by_curve_name calls EC_GROUP_new_curve_GFp so it's not thread +# safe. We can't use the lock because it would be too slow; instead, we use +# EC_KEY_new and then EC_KEY_set_group which calls EC_GROUP_copy instead which +# is thread safe. +lock = threading.Lock() + + +class BN: + # BN_CTX + class Context: + def __init__(self): + self.ptr = lib.BN_CTX_new() + self.lib = lib # For finalizer + + + def __del__(self): + self.lib.BN_CTX_free(self.ptr) + + + @classmethod + def get(cls): + # Get thread-safe contexf + if not hasattr(thread_local, "bn_ctx"): + thread_local.bn_ctx = cls() + return thread_local.bn_ctx.ptr + + + def __init__(self, value=None, link_only=False): + if link_only: + self.bn = value + self._free = False + else: + if value is None: + self.bn = lib.BN_new() + self._free = True + elif isinstance(value, int) and value < 256: + self.bn = lib.BN_new() + lib.BN_clear(self.bn) + lib.BN_add_word(self.bn, value) + self._free = True + else: + if isinstance(value, int): + value = value.to_bytes(128, "big") + self.bn = lib.BN_bin2bn(value, len(value), None) + self._free = True + + + def __del__(self): + if self._free: + lib.BN_free(self.bn) + + + def bytes(self, length=None): + buf = ctypes.create_string_buffer((len(self) + 7) // 8) + lib.BN_bn2bin(self.bn, buf) + buf = bytes(buf) + if length is None: + return buf + else: + if length < len(buf): + raise ValueError("Too little space for BN") + return b"\x00" * (length - len(buf)) + buf + + def __int__(self): + value = 0 + for byte in self.bytes(): + value = value * 256 + byte + return value + + def __len__(self): + return lib.BN_num_bits(self.bn) + + + def inverse(self, modulo): + result = BN() + if not lib.BN_mod_inverse(result.bn, self.bn, modulo.bn, BN.Context.get()): + raise ValueError("Could not compute inverse") + return result + + + def __floordiv__(self, other): + if not isinstance(other, BN): + raise TypeError("Can only divide BN by BN, not {}".format(other)) + result = BN() + if not lib.BN_div(result.bn, None, self.bn, other.bn, BN.Context.get()): + raise ZeroDivisionError("Division by zero") + return result + + def __mod__(self, other): + if not isinstance(other, BN): + raise TypeError("Can only divide BN by BN, not {}".format(other)) + result = BN() + if not lib.BN_div(None, result.bn, self.bn, other.bn, BN.Context.get()): + raise ZeroDivisionError("Division by zero") + return result + + def __add__(self, other): + if not isinstance(other, BN): + raise TypeError("Can only sum BN's, not BN and {}".format(other)) + result = BN() + if not lib.BN_add(result.bn, self.bn, other.bn): + raise ValueError("Could not sum two BN's") + return result + + def __sub__(self, other): + if not isinstance(other, BN): + raise TypeError("Can only subtract BN's, not BN and {}".format(other)) + result = BN() + if not lib.BN_sub(result.bn, self.bn, other.bn): + raise ValueError("Could not subtract BN from BN") + return result + + def __mul__(self, other): + if not isinstance(other, BN): + raise TypeError("Can only multiply BN by BN, not {}".format(other)) + result = BN() + if not lib.BN_mul(result.bn, self.bn, other.bn, BN.Context.get()): + raise ValueError("Could not multiply two BN's") + return result + + def __neg__(self): + return BN(0) - self + + + # A dirty but nice way to update current BN and free old BN at the same time + def __imod__(self, other): + res = self % other + self.bn, res.bn = res.bn, self.bn + return self + def __iadd__(self, other): + res = self + other + self.bn, res.bn = res.bn, self.bn + return self + def __isub__(self, other): + res = self - other + self.bn, res.bn = res.bn, self.bn + return self + def __imul__(self, other): + res = self * other + self.bn, res.bn = res.bn, self.bn + return self + + + def cmp(self, other): + if not isinstance(other, BN): + raise TypeError("Can only compare BN with BN, not {}".format(other)) + return lib.BN_cmp(self.bn, other.bn) + + def __eq__(self, other): + return self.cmp(other) == 0 + def __lt__(self, other): + return self.cmp(other) < 0 + def __gt__(self, other): + return self.cmp(other) > 0 + def __ne__(self, other): + return self.cmp(other) != 0 + def __le__(self, other): + return self.cmp(other) <= 0 + def __ge__(self, other): + return self.cmp(other) >= 0 + + + def __repr__(self): + return "".format(int(self)) + + def __str__(self): + return str(int(self)) + + +class EllipticCurveBackend: + def __init__(self, p, n, a, b, g): + bn_ctx = BN.Context.get() + + self.lib = lib # For finalizer + + self.p = BN(p) + self.order = BN(n) + self.a = BN(a) + self.b = BN(b) + self.h = BN((p + n // 2) // n) + + with lock: + # Thread-safety + self.group = lib.EC_GROUP_new_curve_GFp(self.p.bn, self.a.bn, self.b.bn, bn_ctx) + if not self.group: + raise ValueError("Could not create group object") + generator = self._public_key_to_point(g) + lib.EC_GROUP_set_generator(self.group, generator, self.order.bn, self.h.bn) + if not self.group: + raise ValueError("The curve is not supported by OpenSSL") + + self.public_key_length = (len(self.p) + 7) // 8 + + self.is_supported_evp_pkey_ctx = hasattr(lib, "EVP_PKEY_CTX_new") + + + def __del__(self): + self.lib.EC_GROUP_free(self.group) + + + def _private_key_to_ec_key(self, private_key): + # Thread-safety + eckey = lib.EC_KEY_new() + lib.EC_KEY_set_group(eckey, self.group) + if not eckey: + raise ValueError("Failed to allocate EC_KEY") + private_key = BN(private_key) + if not lib.EC_KEY_set_private_key(eckey, private_key.bn): + lib.EC_KEY_free(eckey) + raise ValueError("Invalid private key") + return eckey, private_key + + + def _public_key_to_point(self, public_key): + x = BN(public_key[0]) + y = BN(public_key[1]) + # EC_KEY_set_public_key_affine_coordinates is not supported by + # OpenSSL 1.0.0 so we can't use it + point = lib.EC_POINT_new(self.group) + if not lib.EC_POINT_set_affine_coordinates_GFp(self.group, point, x.bn, y.bn, BN.Context.get()): + raise ValueError("Could not set public key affine coordinates") + return point + + + def _public_key_to_ec_key(self, public_key): + # Thread-safety + eckey = lib.EC_KEY_new() + lib.EC_KEY_set_group(eckey, self.group) + if not eckey: + raise ValueError("Failed to allocate EC_KEY") + try: + # EC_KEY_set_public_key_affine_coordinates is not supported by + # OpenSSL 1.0.0 so we can't use it + point = self._public_key_to_point(public_key) + if not lib.EC_KEY_set_public_key(eckey, point): + raise ValueError("Could not set point") + lib.EC_POINT_free(point) + return eckey + except Exception as e: + lib.EC_KEY_free(eckey) + raise e from None + + + def _point_to_affine(self, point): + # Convert to affine coordinates + x = BN() + y = BN() + if lib.EC_POINT_get_affine_coordinates_GFp(self.group, point, x.bn, y.bn, BN.Context.get()) != 1: + raise ValueError("Failed to convert public key to affine coordinates") + # Convert to binary + if (len(x) + 7) // 8 > self.public_key_length: + raise ValueError("Public key X coordinate is too large") + if (len(y) + 7) // 8 > self.public_key_length: + raise ValueError("Public key Y coordinate is too large") + return x.bytes(self.public_key_length), y.bytes(self.public_key_length) + + + def decompress_point(self, public_key): + point = lib.EC_POINT_new(self.group) + if not point: + raise ValueError("Could not create point") + try: + if not lib.EC_POINT_oct2point(self.group, point, public_key, len(public_key), BN.Context.get()): + raise ValueError("Invalid compressed public key") + return self._point_to_affine(point) + finally: + lib.EC_POINT_free(point) + + + def new_private_key(self): + # Create random key + # Thread-safety + eckey = lib.EC_KEY_new() + lib.EC_KEY_set_group(eckey, self.group) + lib.EC_KEY_generate_key(eckey) + # To big integer + private_key = BN(lib.EC_KEY_get0_private_key(eckey), link_only=True) + # To binary + private_key_buf = private_key.bytes(self.public_key_length) + # Cleanup + lib.EC_KEY_free(eckey) + return private_key_buf + + + def private_to_public(self, private_key): + eckey, private_key = self._private_key_to_ec_key(private_key) + try: + # Derive public key + point = lib.EC_POINT_new(self.group) + try: + if not lib.EC_POINT_mul(self.group, point, private_key.bn, None, None, BN.Context.get()): + raise ValueError("Failed to derive public key") + return self._point_to_affine(point) + finally: + lib.EC_POINT_free(point) + finally: + lib.EC_KEY_free(eckey) + + + def ecdh(self, private_key, public_key): + if not self.is_supported_evp_pkey_ctx: + # Use ECDH_compute_key instead + # Create EC_KEY from private key + eckey, _ = self._private_key_to_ec_key(private_key) + try: + # Create EC_POINT from public key + point = self._public_key_to_point(public_key) + try: + key = ctypes.create_string_buffer(self.public_key_length) + if lib.ECDH_compute_key(key, self.public_key_length, point, eckey, None) == -1: + raise ValueError("Could not compute shared secret") + return bytes(key) + finally: + lib.EC_POINT_free(point) + finally: + lib.EC_KEY_free(eckey) + + # Private key: + # Create EC_KEY + eckey, _ = self._private_key_to_ec_key(private_key) + try: + # Convert to EVP_PKEY + pkey = lib.EVP_PKEY_new() + if not pkey: + raise ValueError("Could not create private key object") + try: + lib.EVP_PKEY_set1_EC_KEY(pkey, eckey) + + # Public key: + # Create EC_KEY + peer_eckey = self._public_key_to_ec_key(public_key) + try: + # Convert to EVP_PKEY + peer_pkey = lib.EVP_PKEY_new() + if not peer_pkey: + raise ValueError("Could not create public key object") + try: + lib.EVP_PKEY_set1_EC_KEY(peer_pkey, peer_eckey) + + # Create context + ctx = lib.EVP_PKEY_CTX_new(pkey, None) + if not ctx: + raise ValueError("Could not create EVP context") + try: + if lib.EVP_PKEY_derive_init(ctx) != 1: + raise ValueError("Could not initialize key derivation") + if not lib.EVP_PKEY_derive_set_peer(ctx, peer_pkey): + raise ValueError("Could not set peer") + + # Actually derive + key_len = ctypes.c_int(0) + lib.EVP_PKEY_derive(ctx, None, ctypes.byref(key_len)) + key = ctypes.create_string_buffer(key_len.value) + lib.EVP_PKEY_derive(ctx, key, ctypes.byref(key_len)) + + return bytes(key) + finally: + lib.EVP_PKEY_CTX_free(ctx) + finally: + lib.EVP_PKEY_free(peer_pkey) + finally: + lib.EC_KEY_free(peer_eckey) + finally: + lib.EVP_PKEY_free(pkey) + finally: + lib.EC_KEY_free(eckey) + + + def _subject_to_bn(self, subject): + return BN(subject[:(len(self.order) + 7) // 8]) + + + def sign(self, subject, private_key, recoverable, is_compressed, entropy): + z = self._subject_to_bn(subject) + private_key = BN(private_key) + k = BN(entropy) + + rp = lib.EC_POINT_new(self.group) + bn_ctx = BN.Context.get() + try: + # Fix Minerva + k1 = k + self.order + k2 = k1 + self.order + if len(k1) == len(k2): + k = k2 + else: + k = k1 + if not lib.EC_POINT_mul(self.group, rp, k.bn, None, None, bn_ctx): + raise ValueError("Could not generate R") + # Convert to affine coordinates + rx = BN() + ry = BN() + if lib.EC_POINT_get_affine_coordinates_GFp(self.group, rp, rx.bn, ry.bn, bn_ctx) != 1: + raise ValueError("Failed to convert R to affine coordinates") + r = rx % self.order + if r == BN(0): + raise ValueError("Invalid k") + # Calculate s = k^-1 * (z + r * private_key) mod n + s = (k.inverse(self.order) * (z + r * private_key)) % self.order + if s == BN(0): + raise ValueError("Invalid k") + + inverted = False + if s * BN(2) >= self.order: + s = self.order - s + inverted = True + + r_buf = r.bytes(self.public_key_length) + s_buf = s.bytes(self.public_key_length) + if recoverable: + # Generate recid + recid = int(ry % BN(2)) ^ inverted + # The line below is highly unlikely to matter in case of + # secp256k1 but might make sense for other curves + recid += 2 * int(rx // self.order) + if is_compressed: + return bytes([31 + recid]) + r_buf + s_buf + else: + if recid >= 4: + raise ValueError("Too big recovery ID, use compressed address instead") + return bytes([27 + recid]) + r_buf + s_buf + else: + return r_buf + s_buf + finally: + lib.EC_POINT_free(rp) + + + def recover(self, signature, subject): + recid = signature[0] - 27 if signature[0] < 31 else signature[0] - 31 + r = BN(signature[1:self.public_key_length + 1]) + s = BN(signature[self.public_key_length + 1:]) + + # Verify bounds + if r >= self.order: + raise ValueError("r is out of bounds") + if s >= self.order: + raise ValueError("s is out of bounds") + + bn_ctx = BN.Context.get() + + z = self._subject_to_bn(subject) + + rinv = r.inverse(self.order) + u1 = (-z * rinv) % self.order + u2 = (s * rinv) % self.order + + # Recover R + rx = r + BN(recid // 2) * self.order + if rx >= self.p: + raise ValueError("Rx is out of bounds") + rp = lib.EC_POINT_new(self.group) + if not rp: + raise ValueError("Could not create R") + try: + init_buf = b"\x02" + rx.bytes(self.public_key_length) + if not lib.EC_POINT_oct2point(self.group, rp, init_buf, len(init_buf), bn_ctx): + raise ValueError("Could not use Rx to initialize point") + ry = BN() + if lib.EC_POINT_get_affine_coordinates_GFp(self.group, rp, None, ry.bn, bn_ctx) != 1: + raise ValueError("Failed to convert R to affine coordinates") + if int(ry % BN(2)) != recid % 2: + # Fix Ry sign + ry = self.p - ry + if lib.EC_POINT_set_affine_coordinates_GFp(self.group, rp, rx.bn, ry.bn, bn_ctx) != 1: + raise ValueError("Failed to update R coordinates") + + # Recover public key + result = lib.EC_POINT_new(self.group) + if not result: + raise ValueError("Could not create point") + try: + if not lib.EC_POINT_mul(self.group, result, u1.bn, rp, u2.bn, bn_ctx): + raise ValueError("Could not recover public key") + return self._point_to_affine(result) + finally: + lib.EC_POINT_free(result) + finally: + lib.EC_POINT_free(rp) + + + def verify(self, signature, subject, public_key): + r_raw = signature[:self.public_key_length] + r = BN(r_raw) + s = BN(signature[self.public_key_length:]) + if r >= self.order: + raise ValueError("r is out of bounds") + if s >= self.order: + raise ValueError("s is out of bounds") + + bn_ctx = BN.Context.get() + + z = self._subject_to_bn(subject) + + pub_p = lib.EC_POINT_new(self.group) + if not pub_p: + raise ValueError("Could not create public key point") + try: + init_buf = b"\x04" + public_key[0] + public_key[1] + if not lib.EC_POINT_oct2point(self.group, pub_p, init_buf, len(init_buf), bn_ctx): + raise ValueError("Could initialize point") + + sinv = s.inverse(self.order) + u1 = (z * sinv) % self.order + u2 = (r * sinv) % self.order + + # Recover public key + result = lib.EC_POINT_new(self.group) + if not result: + raise ValueError("Could not create point") + try: + if not lib.EC_POINT_mul(self.group, result, u1.bn, pub_p, u2.bn, bn_ctx): + raise ValueError("Could not recover public key") + if BN(self._point_to_affine(result)[0]) % self.order != r: + raise ValueError("Invalid signature") + return True + finally: + lib.EC_POINT_free(result) + finally: + lib.EC_POINT_free(pub_p) + + + def derive_child(self, seed, child): + # Round 1 + h = hmac.new(key=b"Bitcoin seed", msg=seed, digestmod="sha512").digest() + private_key1 = h[:32] + x, y = self.private_to_public(private_key1) + public_key1 = bytes([0x02 + (y[-1] % 2)]) + x + private_key1 = BN(private_key1) + + # Round 2 + child_bytes = [] + for _ in range(4): + child_bytes.append(child & 255) + child >>= 8 + child_bytes = bytes(child_bytes[::-1]) + msg = public_key1 + child_bytes + h = hmac.new(key=h[32:], msg=msg, digestmod="sha512").digest() + private_key2 = BN(h[:32]) + + return ((private_key1 + private_key2) % self.order).bytes(self.public_key_length) + + + @classmethod + def get_backend(cls): + return openssl_backend + + +ecc = ECC(EllipticCurveBackend, aes) diff --git a/src/lib/sslcrypto/openssl/library.py b/src/lib/sslcrypto/openssl/library.py new file mode 100644 index 00000000..47bedc3a --- /dev/null +++ b/src/lib/sslcrypto/openssl/library.py @@ -0,0 +1,98 @@ +import os +import sys +import ctypes +import ctypes.util +from .discovery import discover as user_discover + + +# Disable false-positive _MEIPASS +# pylint: disable=no-member,protected-access + +# Discover OpenSSL library +def discover_paths(): + # Search local files first + if "win" in sys.platform: + # Windows + names = [ + "libeay32.dll" + ] + openssl_paths = [os.path.abspath(path) for path in names] + if hasattr(sys, "_MEIPASS"): + openssl_paths += [os.path.join(sys._MEIPASS, path) for path in openssl_paths] + openssl_paths.append(ctypes.util.find_library("libeay32")) + elif "darwin" in sys.platform: + # Mac OS + names = [ + "libcrypto.dylib", + "libcrypto.1.1.0.dylib", + "libcrypto.1.0.2.dylib", + "libcrypto.1.0.1.dylib", + "libcrypto.1.0.0.dylib", + "libcrypto.0.9.8.dylib" + ] + openssl_paths = [os.path.abspath(path) for path in names] + openssl_paths += names + openssl_paths += [ + "/usr/local/opt/openssl/lib/libcrypto.dylib" + ] + if hasattr(sys, "_MEIPASS") and "RESOURCEPATH" in os.environ: + openssl_paths += [ + os.path.join(os.environ["RESOURCEPATH"], "..", "Frameworks", name) + for name in names + ] + openssl_paths.append(ctypes.util.find_library("ssl")) + else: + # Linux, BSD and such + names = [ + "libcrypto.so", + "libssl.so", + "libcrypto.so.1.1.0", + "libssl.so.1.1.0", + "libcrypto.so.1.0.2", + "libssl.so.1.0.2", + "libcrypto.so.1.0.1", + "libssl.so.1.0.1", + "libcrypto.so.1.0.0", + "libssl.so.1.0.0", + "libcrypto.so.0.9.8", + "libssl.so.0.9.8" + ] + openssl_paths = [os.path.abspath(path) for path in names] + openssl_paths += names + if hasattr(sys, "_MEIPASS"): + openssl_paths += [os.path.join(sys._MEIPASS, path) for path in names] + openssl_paths.append(ctypes.util.find_library("ssl")) + lst = user_discover() + if isinstance(lst, str): + lst = [lst] + elif not lst: + lst = [] + return lst + openssl_paths + + +def discover_library(): + for path in discover_paths(): + if path: + try: + return ctypes.CDLL(path) + except OSError: + pass + raise OSError("OpenSSL is unavailable") + + +lib = discover_library() + +# Initialize internal state +try: + lib.OPENSSL_add_all_algorithms_conf() +except AttributeError: + pass + +try: + lib.OpenSSL_version.restype = ctypes.c_char_p + openssl_backend = lib.OpenSSL_version(0).decode() +except AttributeError: + lib.SSLeay_version.restype = ctypes.c_char_p + openssl_backend = lib.SSLeay_version(0).decode() + +openssl_backend += " at " + lib._name diff --git a/src/lib/sslcrypto/openssl/rsa.py b/src/lib/sslcrypto/openssl/rsa.py new file mode 100644 index 00000000..afd8b51c --- /dev/null +++ b/src/lib/sslcrypto/openssl/rsa.py @@ -0,0 +1,11 @@ +# pylint: disable=too-few-public-methods + +from .library import openssl_backend + + +class RSA: + def get_backend(self): + return openssl_backend + + +rsa = RSA() diff --git a/src/lib/subtl/LICENCE b/src/lib/subtl/LICENCE new file mode 100644 index 00000000..a73bf40a --- /dev/null +++ b/src/lib/subtl/LICENCE @@ -0,0 +1,23 @@ +Copyright (c) 2012, Packetloop. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Packetloop nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/lib/subtl/README.md b/src/lib/subtl/README.md new file mode 100644 index 00000000..80ae983e --- /dev/null +++ b/src/lib/subtl/README.md @@ -0,0 +1,28 @@ +# subtl + +## Overview + +SUBTL is a **s**imple **U**DP **B**itTorrent **t**racker **l**ibrary for Python, licenced under the modified BSD license. + +## Example + +This short example will list a few IP Addresses from a certain hash: + + from subtl import UdpTrackerClient + utc = UdpTrackerClient('tracker.openbittorrent.com', 80) + utc.connect() + if not utc.poll_once(): + raise Exception('Could not connect') + print('Success!') + + utc.announce(info_hash='089184ED52AA37F71801391C451C5D5ADD0D9501') + data = utc.poll_once() + if not data: + raise Exception('Could not announce') + for a in data['response']['peers']: + print(a) + +## Caveats + + * There is no automatic retrying of sending packets yet. + * This library won't download torrent files--it is simply a tracker client. diff --git a/src/lib/subtl/__init__.py b/src/lib/subtl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/lib/subtl/subtl.py b/src/lib/subtl/subtl.py new file mode 100644 index 00000000..cd8c5b2c --- /dev/null +++ b/src/lib/subtl/subtl.py @@ -0,0 +1,209 @@ +''' +Based on the specification at http://bittorrent.org/beps/bep_0015.html +''' +import binascii +import random +import struct +import time +import socket +from collections import defaultdict + + +__version__ = '0.0.1' + +CONNECT = 0 +ANNOUNCE = 1 +SCRAPE = 2 +ERROR = 3 + + +class UdpTrackerClientException(Exception): + pass + + +class UdpTrackerClient: + + def __init__(self, host, port): + self.host = host + self.port = port + self.peer_port = 6881 + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.conn_id = 0x41727101980 + self.transactions = {} + self.peer_id = self._generate_peer_id() + self.timeout = 9 + + def connect(self): + return self._send(CONNECT) + + def announce(self, **kwargs): + if not kwargs: + raise UdpTrackerClientException('arguments missing') + args = { + 'peer_id': self.peer_id, + 'downloaded': 0, + 'left': 0, + 'uploaded': 0, + 'event': 0, + 'key': 0, + 'num_want': 10, + 'ip_address': 0, + 'port': self.peer_port, + } + args.update(kwargs) + + fields = 'info_hash peer_id downloaded left uploaded event ' \ + 'ip_address key num_want port' + + # Check and raise if missing fields + self._check_fields(args, fields) + + # Humans tend to use hex representations of the hash. Wasteful humans. + args['info_hash'] = args['info_hash'] + + values = [args[a] for a in fields.split()] + values[1] = values[1].encode("utf8") + payload = struct.pack('!20s20sQQQLLLLH', *values) + return self._send(ANNOUNCE, payload) + + def scrape(self, info_hash_list): + if len(info_hash_list) > 74: + raise UdpTrackerClientException('Max info_hashes is 74') + + payload = '' + for info_hash in info_hash_list: + payload += info_hash + + trans = self._send(SCRAPE, payload) + trans['sent_hashes'] = info_hash_list + return trans + + def poll_once(self): + self.sock.settimeout(self.timeout) + try: + response = self.sock.recv(10240) + except socket.timeout: + return + + header = response[:8] + payload = response[8:] + action, trans_id = struct.unpack('!LL', header) + try: + trans = self.transactions[trans_id] + except KeyError: + self.error('transaction_id not found') + return + trans['response'] = self._process_response(action, payload, trans) + trans['completed'] = True + del self.transactions[trans_id] + return trans + + def error(self, message): + raise Exception('error: {}'.format(message)) + + def _send(self, action, payload=None): + if not payload: + payload = b'' + trans_id, header = self._request_header(action) + self.transactions[trans_id] = trans = { + 'action': action, + 'time': time.time(), + 'payload': payload, + 'completed': False, + } + self.sock.connect((self.host, self.port)) + self.sock.send(header + payload) + return trans + + def _request_header(self, action): + trans_id = random.randint(0, (1 << 32) - 1) + return trans_id, struct.pack('!QLL', self.conn_id, action, trans_id) + + def _process_response(self, action, payload, trans): + if action == CONNECT: + return self._process_connect(payload, trans) + elif action == ANNOUNCE: + return self._process_announce(payload, trans) + elif action == SCRAPE: + return self._process_scrape(payload, trans) + elif action == ERROR: + return self._process_error(payload, trans) + else: + raise UdpTrackerClientException( + 'Unknown action response: {}'.format(action)) + + def _process_connect(self, payload, trans): + self.conn_id = struct.unpack('!Q', payload)[0] + return self.conn_id + + def _process_announce(self, payload, trans): + response = {} + + info_struct = '!LLL' + info_size = struct.calcsize(info_struct) + info = payload[:info_size] + interval, leechers, seeders = struct.unpack(info_struct, info) + + peer_data = payload[info_size:] + peer_struct = '!LH' + peer_size = struct.calcsize(peer_struct) + peer_count = int(len(peer_data) / peer_size) + peers = [] + + for peer_offset in range(peer_count): + off = peer_size * peer_offset + peer = peer_data[off:off + peer_size] + addr, port = struct.unpack(peer_struct, peer) + peers.append({ + 'addr': socket.inet_ntoa(struct.pack('!L', addr)), + 'port': port, + }) + + return { + 'interval': interval, + 'leechers': leechers, + 'seeders': seeders, + 'peers': peers, + } + + def _process_scrape(self, payload, trans): + info_struct = '!LLL' + info_size = struct.calcsize(info_struct) + info_count = len(payload) / info_size + hashes = trans['sent_hashes'] + response = {} + for info_offset in range(info_count): + off = info_size * info_offset + info = payload[off:off + info_size] + seeders, completed, leechers = struct.unpack(info_struct, info) + response[hashes[info_offset]] = { + 'seeders': seeders, + 'completed': completed, + 'leechers': leechers, + } + return response + + def _process_error(self, payload, trans): + ''' + I haven't seen this action type be sent from a tracker, but I've left + it here for the possibility. + ''' + self.error(payload) + return False + + def _generate_peer_id(self): + '''http://www.bittorrent.org/beps/bep_0020.html''' + peer_id = '-PU' + __version__.replace('.', '-') + '-' + remaining = 20 - len(peer_id) + numbers = [str(random.randint(0, 9)) for _ in range(remaining)] + peer_id += ''.join(numbers) + assert(len(peer_id) == 20) + return peer_id + + def _check_fields(self, args, fields): + for f in fields: + try: + args.get(f) + except KeyError: + raise UdpTrackerClientException('field missing: {}'.format(f)) + diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..ec90f4d9 --- /dev/null +++ b/src/main.py @@ -0,0 +1,603 @@ +# Included modules +import os +import sys +import stat +import time +import logging + +startup_errors = [] +def startupError(msg): + startup_errors.append(msg) + print("Startup error: %s" % msg) + +# Third party modules +import gevent +if gevent.version_info.major <= 1: # Workaround for random crash when libuv used with threads + try: + if "libev" not in str(gevent.config.loop): + gevent.config.loop = "libev-cext" + except Exception as err: + startupError("Unable to switch gevent loop to libev: %s" % err) + +import gevent.monkey +gevent.monkey.patch_all(thread=False, subprocess=False) + +update_after_shutdown = False # If set True then update and restart zeronet after main loop ended +restart_after_shutdown = False # If set True then restart zeronet after main loop ended + +# Load config +from Config import config +config.parse(silent=True) # Plugins need to access the configuration +if not config.arguments: # Config parse failed, show the help screen and exit + config.parse() + +if not os.path.isdir(config.data_dir): + os.mkdir(config.data_dir) + try: + os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except Exception as err: + startupError("Can't change permission of %s: %s" % (config.data_dir, err)) + +if not os.path.isfile("%s/sites.json" % config.data_dir): + open("%s/sites.json" % config.data_dir, "w").write("{}") +if not os.path.isfile("%s/users.json" % config.data_dir): + open("%s/users.json" % config.data_dir, "w").write("{}") + +if config.action == "main": + from util import helper + try: + lock = helper.openLocked("%s/lock.pid" % config.data_dir, "w") + lock.write("%s" % os.getpid()) + except BlockingIOError as err: + startupError("Can't open lock file, your ZeroNet client is probably already running, exiting... (%s)" % err) + if config.open_browser and config.open_browser != "False": + print("Opening browser: %s...", config.open_browser) + import webbrowser + try: + if config.open_browser == "default_browser": + browser = webbrowser.get() + else: + browser = webbrowser.get(config.open_browser) + browser.open("http://%s:%s/%s" % ( + config.ui_ip if config.ui_ip != "*" else "127.0.0.1", config.ui_port, config.homepage + ), new=2) + except Exception as err: + startupError("Error starting browser: %s" % err) + sys.exit() + +config.initLogging() + +# Debug dependent configuration +from Debug import DebugHook + +# Load plugins +from Plugin import PluginManager +PluginManager.plugin_manager.loadPlugins() +config.loadPlugins() +config.parse() # Parse again to add plugin configuration options + +# Log current config +logging.debug("Config: %s" % config) + +# Modify stack size on special hardwares +if config.stack_size: + import threading + threading.stack_size(config.stack_size) + +# Use pure-python implementation of msgpack to save CPU +if config.msgpack_purepython: + os.environ["MSGPACK_PUREPYTHON"] = "True" + +# Fix console encoding on Windows +if sys.platform.startswith("win"): + import subprocess + try: + chcp_res = subprocess.check_output("chcp 65001", shell=True).decode(errors="ignore").strip() + logging.debug("Changed console encoding to utf8: %s" % chcp_res) + except Exception as err: + logging.error("Error changing console encoding to utf8: %s" % err) + +# Socket monkey patch +if config.proxy: + from util import SocksProxy + import urllib.request + logging.info("Patching sockets to socks proxy: %s" % config.proxy) + if config.fileserver_ip == "*": + config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost + config.disable_udp = True # UDP not supported currently with proxy + SocksProxy.monkeyPatch(*config.proxy.split(":")) +elif config.tor == "always": + from util import SocksProxy + import urllib.request + logging.info("Patching sockets to tor socks proxy: %s" % config.tor_proxy) + if config.fileserver_ip == "*": + config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost + SocksProxy.monkeyPatch(*config.tor_proxy.split(":")) + config.disable_udp = True +elif config.bind: + bind = config.bind + if ":" not in config.bind: + bind += ":0" + from util import helper + helper.socketBindMonkeyPatch(*bind.split(":")) + +# -- Actions -- + + +@PluginManager.acceptPlugins +class Actions(object): + def call(self, function_name, kwargs): + logging.info("Version: %s r%s, Python %s, Gevent: %s" % (config.version, config.rev, sys.version, gevent.__version__)) + + func = getattr(self, function_name, None) + back = func(**kwargs) + if back: + print(back) + + # Default action: Start serving UiServer and FileServer + def main(self): + global ui_server, file_server + from File import FileServer + from Ui import UiServer + logging.info("Creating FileServer....") + file_server = FileServer() + logging.info("Creating UiServer....") + ui_server = UiServer() + file_server.ui_server = ui_server + + for startup_error in startup_errors: + logging.error("Startup error: %s" % startup_error) + + logging.info("Removing old SSL certs...") + from Crypt import CryptConnection + CryptConnection.manager.removeCerts() + + logging.info("Starting servers....") + gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)]) + logging.info("All server stopped") + + # Site commands + + def siteCreate(self, use_master_seed=True): + logging.info("Generating new privatekey (use_master_seed: %s)..." % config.use_master_seed) + from Crypt import CryptBitcoin + if use_master_seed: + from User import UserManager + user = UserManager.user_manager.get() + if not user: + user = UserManager.user_manager.create() + address, address_index, site_data = user.getNewSiteData() + privatekey = site_data["privatekey"] + logging.info("Generated using master seed from users.json, site index: %s" % address_index) + else: + privatekey = CryptBitcoin.newPrivatekey() + address = CryptBitcoin.privatekeyToAddress(privatekey) + logging.info("----------------------------------------------------------------------") + logging.info("Site private key: %s" % privatekey) + logging.info(" !!! ^ Save it now, required to modify the site ^ !!!") + logging.info("Site address: %s" % address) + logging.info("----------------------------------------------------------------------") + + while True and not config.batch and not use_master_seed: + if input("? Have you secured your private key? (yes, no) > ").lower() == "yes": + break + else: + logging.info("Please, secure it now, you going to need it to modify your site!") + + logging.info("Creating directory structure...") + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + os.mkdir("%s/%s" % (config.data_dir, address)) + open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address) + + logging.info("Creating content.json...") + site = Site(address) + extend = {"postmessage_nonce_security": True} + if use_master_seed: + extend["address_index"] = address_index + + site.content_manager.sign(privatekey=privatekey, extend=extend) + site.settings["own"] = True + site.saveSettings() + + logging.info("Site created!") + + def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False, remove_missing_optional=False): + from Site.Site import Site + from Site import SiteManager + from Debug import Debug + SiteManager.site_manager.load() + logging.info("Signing site: %s..." % address) + site = Site(address, allow_create=False) + + if not privatekey: # If no privatekey defined + from User import UserManager + user = UserManager.user_manager.get() + if user: + site_data = user.getSiteData(address) + privatekey = site_data.get("privatekey") + else: + privatekey = None + if not privatekey: + # Not found in users.json, ask from console + import getpass + privatekey = getpass.getpass("Private key (input hidden):") + try: + succ = site.content_manager.sign( + inner_path=inner_path, privatekey=privatekey, + update_changed_files=True, remove_missing_optional=remove_missing_optional + ) + except Exception as err: + logging.error("Sign error: %s" % Debug.formatException(err)) + succ = False + if succ and publish: + self.sitePublish(address, inner_path=inner_path) + + def siteVerify(self, address): + import time + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + s = time.time() + logging.info("Verifing site: %s..." % address) + site = Site(address) + bad_files = [] + + for content_inner_path in site.content_manager.contents: + s = time.time() + logging.info("Verifing %s signature..." % content_inner_path) + err = None + try: + file_correct = site.content_manager.verifyFile( + content_inner_path, site.storage.open(content_inner_path, "rb"), ignore_same=False + ) + except Exception as exp: + file_correct = False + err = exp + + if file_correct is True: + logging.info("[OK] %s (Done in %.3fs)" % (content_inner_path, time.time() - s)) + else: + logging.error("[ERROR] %s: invalid file: %s!" % (content_inner_path, err)) + input("Continue?") + bad_files += content_inner_path + + logging.info("Verifying site files...") + bad_files += site.storage.verifyFiles()["bad_files"] + if not bad_files: + logging.info("[OK] All file sha512sum matches! (%.3fs)" % (time.time() - s)) + else: + logging.error("[ERROR] Error during verifying site files!") + + def dbRebuild(self, address): + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + logging.info("Rebuilding site sql cache: %s..." % address) + site = SiteManager.site_manager.get(address) + s = time.time() + try: + site.storage.rebuildDb() + logging.info("Done in %.3fs" % (time.time() - s)) + except Exception as err: + logging.error(err) + + def dbQuery(self, address, query): + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + import json + site = Site(address) + result = [] + for row in site.storage.query(query): + result.append(dict(row)) + print(json.dumps(result, indent=4)) + + def siteAnnounce(self, address): + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + logging.info("Opening a simple connection server") + global file_server + from File import FileServer + file_server = FileServer("127.0.0.1", 1234) + file_server.start() + + logging.info("Announcing site %s to tracker..." % address) + site = Site(address) + + s = time.time() + site.announce() + print("Response time: %.3fs" % (time.time() - s)) + print(site.peers) + + def siteDownload(self, address): + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + logging.info("Opening a simple connection server") + global file_server + from File import FileServer + file_server = FileServer("127.0.0.1", 1234) + file_server_thread = gevent.spawn(file_server.start, check_sites=False) + + site = Site(address) + + on_completed = gevent.event.AsyncResult() + + def onComplete(evt): + evt.set(True) + + site.onComplete.once(lambda: onComplete(on_completed)) + print("Announcing...") + site.announce() + + s = time.time() + print("Downloading...") + site.downloadContent("content.json", check_modifications=True) + + print("Downloaded in %.3fs" % (time.time()-s)) + + def siteNeedFile(self, address, inner_path): + from Site.Site import Site + from Site import SiteManager + SiteManager.site_manager.load() + + def checker(): + while 1: + s = time.time() + time.sleep(1) + print("Switch time:", time.time() - s) + gevent.spawn(checker) + + logging.info("Opening a simple connection server") + global file_server + from File import FileServer + file_server = FileServer("127.0.0.1", 1234) + file_server_thread = gevent.spawn(file_server.start, check_sites=False) + + site = Site(address) + site.announce() + print(site.needFile(inner_path, update=True)) + + def siteCmd(self, address, cmd, parameters): + import json + from Site import SiteManager + + site = SiteManager.site_manager.get(address) + + if not site: + logging.error("Site not found: %s" % address) + return None + + ws = self.getWebsocket(site) + + ws.send(json.dumps({"cmd": cmd, "params": parameters, "id": 1})) + res_raw = ws.recv() + + try: + res = json.loads(res_raw) + except Exception as err: + return {"error": "Invalid result: %s" % err, "res_raw": res_raw} + + if "result" in res: + return res["result"] + else: + return res + + def getWebsocket(self, site): + import websocket + + ws_address = "ws://%s:%s/Websocket?wrapper_key=%s" % (config.ui_ip, config.ui_port, site.settings["wrapper_key"]) + logging.info("Connecting to %s" % ws_address) + ws = websocket.create_connection(ws_address) + return ws + + def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json"): + global file_server + from Site.Site import Site + from Site import SiteManager + from File import FileServer # We need fileserver to handle incoming file requests + from Peer import Peer + file_server = FileServer() + site = SiteManager.site_manager.get(address) + logging.info("Loading site...") + site.settings["serving"] = True # Serving the site even if its disabled + + try: + ws = self.getWebsocket(site) + logging.info("Sending siteReload") + self.siteCmd(address, "siteReload", inner_path) + + logging.info("Sending sitePublish") + self.siteCmd(address, "sitePublish", {"inner_path": inner_path, "sign": False}) + logging.info("Done.") + + except Exception as err: + logging.info("Can't connect to local websocket client: %s" % err) + logging.info("Creating FileServer....") + file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity + time.sleep(0.001) + + # Started fileserver + file_server.portCheck() + if peer_ip: # Announce ip specificed + site.addPeer(peer_ip, peer_port) + else: # Just ask the tracker + logging.info("Gathering peers from tracker") + site.announce() # Gather peers + published = site.publish(10, inner_path) # Push to peers + if published > 0: + time.sleep(3) + logging.info("Serving files (max 60s)...") + gevent.joinall([file_server_thread], timeout=60) + logging.info("Done.") + else: + logging.info("No peers found, sitePublish command only works if you already have visitors serving your site") + + # Crypto commands + def cryptPrivatekeyToAddress(self, privatekey=None): + from Crypt import CryptBitcoin + if not privatekey: # If no privatekey in args then ask it now + import getpass + privatekey = getpass.getpass("Private key (input hidden):") + + print(CryptBitcoin.privatekeyToAddress(privatekey)) + + def cryptSign(self, message, privatekey): + from Crypt import CryptBitcoin + print(CryptBitcoin.sign(message, privatekey)) + + def cryptVerify(self, message, sign, address): + from Crypt import CryptBitcoin + print(CryptBitcoin.verify(message, address, sign)) + + def cryptGetPrivatekey(self, master_seed, site_address_index=None): + from Crypt import CryptBitcoin + if len(master_seed) != 64: + logging.error("Error: Invalid master seed length: %s (required: 64)" % len(master_seed)) + return False + privatekey = CryptBitcoin.hdPrivatekey(master_seed, site_address_index) + print("Requested private key: %s" % privatekey) + + # Peer + def peerPing(self, peer_ip, peer_port=None): + if not peer_port: + peer_port = 15441 + logging.info("Opening a simple connection server") + global file_server + from Connection import ConnectionServer + file_server = ConnectionServer("127.0.0.1", 1234) + file_server.start(check_connections=False) + from Crypt import CryptConnection + CryptConnection.manager.loadCerts() + + from Peer import Peer + logging.info("Pinging 5 times peer: %s:%s..." % (peer_ip, int(peer_port))) + s = time.time() + peer = Peer(peer_ip, peer_port) + peer.connect() + + if not peer.connection: + print("Error: Can't connect to peer (connection error: %s)" % peer.connection_error) + return False + if "shared_ciphers" in dir(peer.connection.sock): + print("Shared ciphers:", peer.connection.sock.shared_ciphers()) + if "cipher" in dir(peer.connection.sock): + print("Cipher:", peer.connection.sock.cipher()[0]) + if "version" in dir(peer.connection.sock): + print("TLS version:", peer.connection.sock.version()) + print("Connection time: %.3fs (connection error: %s)" % (time.time() - s, peer.connection_error)) + + for i in range(5): + ping_delay = peer.ping() + print("Response time: %.3fs" % ping_delay) + time.sleep(1) + peer.remove() + print("Reconnect test...") + peer = Peer(peer_ip, peer_port) + for i in range(5): + ping_delay = peer.ping() + print("Response time: %.3fs" % ping_delay) + time.sleep(1) + + def peerGetFile(self, peer_ip, peer_port, site, filename, benchmark=False): + logging.info("Opening a simple connection server") + global file_server + from Connection import ConnectionServer + file_server = ConnectionServer("127.0.0.1", 1234) + file_server.start(check_connections=False) + from Crypt import CryptConnection + CryptConnection.manager.loadCerts() + + from Peer import Peer + logging.info("Getting %s/%s from peer: %s:%s..." % (site, filename, peer_ip, peer_port)) + peer = Peer(peer_ip, peer_port) + s = time.time() + if benchmark: + for i in range(10): + peer.getFile(site, filename), + print("Response time: %.3fs" % (time.time() - s)) + input("Check memory") + else: + print(peer.getFile(site, filename).read()) + + def peerCmd(self, peer_ip, peer_port, cmd, parameters): + logging.info("Opening a simple connection server") + global file_server + from Connection import ConnectionServer + file_server = ConnectionServer() + file_server.start(check_connections=False) + from Crypt import CryptConnection + CryptConnection.manager.loadCerts() + + from Peer import Peer + peer = Peer(peer_ip, peer_port) + + import json + if parameters: + parameters = json.loads(parameters.replace("'", '"')) + else: + parameters = {} + try: + res = peer.request(cmd, parameters) + print(json.dumps(res, indent=2, ensure_ascii=False)) + except Exception as err: + print("Unknown response (%s): %s" % (err, res)) + + def getConfig(self): + import json + print(json.dumps(config.getServerInfo(), indent=2, ensure_ascii=False)) + + def test(self, test_name, *args, **kwargs): + import types + def funcToName(func_name): + test_name = func_name.replace("test", "") + return test_name[0].lower() + test_name[1:] + + test_names = [funcToName(name) for name in dir(self) if name.startswith("test") and name != "test"] + if not test_name: + # No test specificed, list tests + print("\nNo test specified, possible tests:") + for test_name in test_names: + func_name = "test" + test_name[0].upper() + test_name[1:] + func = getattr(self, func_name) + if func.__doc__: + print("- %s: %s" % (test_name, func.__doc__.strip())) + else: + print("- %s" % test_name) + return None + + # Run tests + func_name = "test" + test_name[0].upper() + test_name[1:] + if hasattr(self, func_name): + func = getattr(self, func_name) + print("- Running test: %s" % test_name, end="") + s = time.time() + ret = func(*args, **kwargs) + if type(ret) is types.GeneratorType: + for progress in ret: + print(progress, end="") + sys.stdout.flush() + print("\n* Test %s done in %.3fs" % (test_name, time.time() - s)) + else: + print("Unknown test: %r (choose from: %s)" % ( + test_name, test_names + )) + + +actions = Actions() +# Starts here when running zeronet.py + + +def start(): + # Call function + action_kwargs = config.getActionArguments() + actions.call(config.action, action_kwargs) diff --git a/src/util/Cached.py b/src/util/Cached.py new file mode 100644 index 00000000..72d60dbc --- /dev/null +++ b/src/util/Cached.py @@ -0,0 +1,68 @@ +import time + + +class Cached(object): + def __init__(self, timeout): + self.cache_db = {} + self.timeout = timeout + + def __call__(self, func): + def wrapper(*args, **kwargs): + key = "%s %s" % (args, kwargs) + cached_value = None + cache_hit = False + if key in self.cache_db: + cache_hit = True + cached_value, time_cached_end = self.cache_db[key] + if time.time() > time_cached_end: + self.cleanupExpired() + cached_value = None + cache_hit = False + + if cache_hit: + return cached_value + else: + cached_value = func(*args, **kwargs) + time_cached_end = time.time() + self.timeout + self.cache_db[key] = (cached_value, time_cached_end) + return cached_value + + wrapper.emptyCache = self.emptyCache + + return wrapper + + def cleanupExpired(self): + for key in list(self.cache_db.keys()): + cached_value, time_cached_end = self.cache_db[key] + if time.time() > time_cached_end: + del(self.cache_db[key]) + + def emptyCache(self): + num = len(self.cache_db) + self.cache_db.clear() + return num + + +if __name__ == "__main__": + from gevent import monkey + monkey.patch_all() + + @Cached(timeout=2) + def calcAdd(a, b): + print("CalcAdd", a, b) + return a + b + + @Cached(timeout=1) + def calcMultiply(a, b): + print("calcMultiply", a, b) + return a * b + + for i in range(5): + print("---") + print("Emptied", calcAdd.emptyCache()) + assert calcAdd(1, 2) == 3 + print("Emptied", calcAdd.emptyCache()) + assert calcAdd(1, 2) == 3 + assert calcAdd(2, 3) == 5 + assert calcMultiply(2, 3) == 6 + time.sleep(1) diff --git a/src/util/Diff.py b/src/util/Diff.py new file mode 100644 index 00000000..53b82c5a --- /dev/null +++ b/src/util/Diff.py @@ -0,0 +1,50 @@ +import io + +import difflib + + +def sumLen(lines): + return sum(map(len, lines)) + + +def diff(old, new, limit=False): + matcher = difflib.SequenceMatcher(None, old, new) + actions = [] + size = 0 + for tag, old_from, old_to, new_from, new_to in matcher.get_opcodes(): + if tag == "insert": + new_line = new[new_from:new_to] + actions.append(("+", new_line)) + size += sum(map(len, new_line)) + elif tag == "equal": + actions.append(("=", sumLen(old[old_from:old_to]))) + elif tag == "delete": + actions.append(("-", sumLen(old[old_from:old_to]))) + elif tag == "replace": + actions.append(("-", sumLen(old[old_from:old_to]))) + new_lines = new[new_from:new_to] + actions.append(("+", new_lines)) + size += sumLen(new_lines) + if limit and size > limit: + return False + return actions + + +def patch(old_f, actions): + new_f = io.BytesIO() + for action, param in actions: + if type(action) is bytes: + action = action.decode() + if action == "=": # Same lines + new_f.write(old_f.read(param)) + elif action == "-": # Delete lines + old_f.seek(param, 1) # Seek from current position + continue + elif action == "+": # Add lines + for add_line in param: + if type(add_line) is str: + add_line = add_line.encode() + new_f.write(add_line) + else: + raise "Unknown action: %s" % action + return new_f diff --git a/src/util/Electrum.py b/src/util/Electrum.py new file mode 100644 index 00000000..112151aa --- /dev/null +++ b/src/util/Electrum.py @@ -0,0 +1,39 @@ +import hashlib +import struct + + +# Electrum, the heck?! + +def bchr(i): + return struct.pack("B", i) + +def encode(val, base, minlen=0): + base, minlen = int(base), int(minlen) + code_string = b"".join([bchr(x) for x in range(256)]) + result = b"" + while val > 0: + index = val % base + result = code_string[index:index + 1] + result + val //= base + return code_string[0:1] * max(minlen - len(result), 0) + result + +def insane_int(x): + x = int(x) + if x < 253: + return bchr(x) + elif x < 65536: + return bchr(253) + encode(x, 256, 2)[::-1] + elif x < 4294967296: + return bchr(254) + encode(x, 256, 4)[::-1] + else: + return bchr(255) + encode(x, 256, 8)[::-1] + + +def magic(message): + return b"\x18Bitcoin Signed Message:\n" + insane_int(len(message)) + message + +def format(message): + return hashlib.sha256(magic(message)).digest() + +def dbl_format(message): + return hashlib.sha256(format(message)).digest() diff --git a/src/util/Event.py b/src/util/Event.py new file mode 100644 index 00000000..9d642736 --- /dev/null +++ b/src/util/Event.py @@ -0,0 +1,55 @@ +# Based on http://stackoverflow.com/a/2022629 + + +class Event(list): + + def __call__(self, *args, **kwargs): + for f in self[:]: + if "once" in dir(f) and f in self: + self.remove(f) + f(*args, **kwargs) + + def __repr__(self): + return "Event(%s)" % list.__repr__(self) + + def once(self, func, name=None): + func.once = True + func.name = None + if name: # Dont function with same name twice + names = [f.name for f in self if "once" in dir(f)] + if name not in names: + func.name = name + self.append(func) + else: + self.append(func) + return self + + +if __name__ == "__main__": + def testBenchmark(): + def say(pre, text): + print("%s Say: %s" % (pre, text)) + + import time + s = time.time() + on_changed = Event() + for i in range(1000): + on_changed.once(lambda pre: say(pre, "once"), "once") + print("Created 1000 once in %.3fs" % (time.time() - s)) + on_changed("#1") + + def testUsage(): + def say(pre, text): + print("%s Say: %s" % (pre, text)) + + on_changed = Event() + on_changed.once(lambda pre: say(pre, "once")) + on_changed.once(lambda pre: say(pre, "once")) + on_changed.once(lambda pre: say(pre, "namedonce"), "namedonce") + on_changed.once(lambda pre: say(pre, "namedonce"), "namedonce") + on_changed.append(lambda pre: say(pre, "always")) + on_changed("#1") + on_changed("#2") + on_changed("#3") + + testBenchmark() diff --git a/src/util/Flag.py b/src/util/Flag.py new file mode 100644 index 00000000..37cfdfba --- /dev/null +++ b/src/util/Flag.py @@ -0,0 +1,22 @@ +from collections import defaultdict + + +class Flag(object): + def __init__(self): + self.valid_flags = set([ + "admin", # Only allowed to run sites with ADMIN permission + "async_run", # Action will be ran async with gevent.spawn + "no_multiuser" # Action disabled if Multiuser plugin running in open proxy mode + ]) + self.db = defaultdict(set) + + def __getattr__(self, key): + def func(f): + if key not in self.valid_flags: + raise Exception("Invalid flag: %s (valid: %s)" % (key, self.valid_flags)) + self.db[f.__name__].add(key) + return f + return func + + +flag = Flag() diff --git a/src/util/GreenletManager.py b/src/util/GreenletManager.py new file mode 100644 index 00000000..e024233d --- /dev/null +++ b/src/util/GreenletManager.py @@ -0,0 +1,24 @@ +import gevent +from Debug import Debug + + +class GreenletManager: + def __init__(self): + self.greenlets = set() + + def spawnLater(self, *args, **kwargs): + greenlet = gevent.spawn_later(*args, **kwargs) + greenlet.link(lambda greenlet: self.greenlets.remove(greenlet)) + self.greenlets.add(greenlet) + return greenlet + + def spawn(self, *args, **kwargs): + greenlet = gevent.spawn(*args, **kwargs) + greenlet.link(lambda greenlet: self.greenlets.remove(greenlet)) + self.greenlets.add(greenlet) + return greenlet + + def stopGreenlets(self, reason="Stopping all greenlets"): + num = len(self.greenlets) + gevent.killall(list(self.greenlets), Debug.createNotifyType(reason), block=False) + return num diff --git a/src/util/Msgpack.py b/src/util/Msgpack.py new file mode 100644 index 00000000..1033f92e --- /dev/null +++ b/src/util/Msgpack.py @@ -0,0 +1,101 @@ +import os +import struct +import io + +import msgpack +import msgpack.fallback + + +def msgpackHeader(size): + if size <= 2 ** 8 - 1: + return b"\xc4" + struct.pack("B", size) + elif size <= 2 ** 16 - 1: + return b"\xc5" + struct.pack(">H", size) + elif size <= 2 ** 32 - 1: + return b"\xc6" + struct.pack(">I", size) + else: + raise Exception("huge binary string") + + +def stream(data, writer): + packer = msgpack.Packer(use_bin_type=True) + writer(packer.pack_map_header(len(data))) + for key, val in data.items(): + writer(packer.pack(key)) + if isinstance(val, io.IOBase): # File obj + max_size = os.fstat(val.fileno()).st_size - val.tell() + size = min(max_size, val.read_bytes) + bytes_left = size + writer(msgpackHeader(size)) + buff = 1024 * 64 + while 1: + writer(val.read(min(bytes_left, buff))) + bytes_left = bytes_left - buff + if bytes_left <= 0: + break + else: # Simple + writer(packer.pack(val)) + return size + + +class FilePart(object): + __slots__ = ("file", "read_bytes", "__class__") + + def __init__(self, *args, **kwargs): + self.file = open(*args, **kwargs) + self.__enter__ == self.file.__enter__ + + def __getattr__(self, attr): + return getattr(self.file, attr) + + def __enter__(self, *args, **kwargs): + return self.file.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.file.__exit__(*args, **kwargs) + + +# Don't try to decode the value of these fields as utf8 +bin_value_keys = ("hashfield_raw", "peers", "peers_ipv6", "peers_onion", "body", "sites", "bin") + + +def objectDecoderHook(obj): + global bin_value_keys + back = {} + for key, val in obj: + if type(key) is bytes: + key = key.decode("utf8") + if key in bin_value_keys or type(val) is not bytes or len(key) >= 64: + back[key] = val + else: + back[key] = val.decode("utf8") + return back + + +def getUnpacker(fallback=False, decode=True): + if fallback: # Pure Python + unpacker = msgpack.fallback.Unpacker + else: + unpacker = msgpack.Unpacker + + extra_kwargs = {"max_buffer_size": 5 * 1024 * 1024} + if msgpack.version[0] >= 1: + extra_kwargs["strict_map_key"] = False + + if decode: # Workaround for backward compatibility: Try to decode bin to str + unpacker = unpacker(raw=True, object_pairs_hook=objectDecoderHook, **extra_kwargs) + else: + unpacker = unpacker(raw=False, **extra_kwargs) + + return unpacker + + +def pack(data, use_bin_type=True): + return msgpack.packb(data, use_bin_type=use_bin_type) + + +def unpack(data, decode=True): + unpacker = getUnpacker(decode=decode) + unpacker.feed(data) + return next(unpacker) + diff --git a/src/util/Noparallel.py b/src/util/Noparallel.py new file mode 100644 index 00000000..4a4a854d --- /dev/null +++ b/src/util/Noparallel.py @@ -0,0 +1,202 @@ +import gevent +import time +from gevent.event import AsyncResult + +from . import ThreadPool + + +class Noparallel: # Only allow function running once in same time + + def __init__(self, blocking=True, ignore_args=False, ignore_class=False, queue=False): + self.threads = {} + self.blocking = blocking # Blocking: Acts like normal function else thread returned + self.queue = queue # Execute again when blocking is done + self.queued = False + self.ignore_args = ignore_args # Block does not depend on function call arguments + self.ignore_class = ignore_class # Block does not depeds on class instance + + def __call__(self, func): + def wrapper(*args, **kwargs): + if not ThreadPool.isMainThread(): + return ThreadPool.main_loop.call(wrapper, *args, **kwargs) + + if self.ignore_class: + key = func # Unique key only by function and class object + elif self.ignore_args: + key = (func, args[0]) # Unique key only by function and class object + else: + key = (func, tuple(args), str(kwargs)) # Unique key for function including parameters + if key in self.threads: # Thread already running (if using blocking mode) + if self.queue: + self.queued = True + thread = self.threads[key] + if self.blocking: + if self.queued: + res = thread.get() # Blocking until its finished + if key in self.threads: + return self.threads[key].get() # Queue finished since started running + self.queued = False + return wrapper(*args, **kwargs) # Run again after the end + else: + return thread.get() # Return the value + + else: # No blocking + if thread.ready(): # Its finished, create a new + thread = gevent.spawn(func, *args, **kwargs) + self.threads[key] = thread + return thread + else: # Still running + return thread + else: # Thread not running + if self.blocking: # Wait for finish + asyncres = AsyncResult() + self.threads[key] = asyncres + try: + res = func(*args, **kwargs) + asyncres.set(res) + self.cleanup(key, asyncres) + return res + except Exception as err: + asyncres.set_exception(err) + self.cleanup(key, asyncres) + raise(err) + else: # No blocking just return the thread + thread = gevent.spawn(func, *args, **kwargs) # Spawning new thread + thread.link(lambda thread: self.cleanup(key, thread)) + self.threads[key] = thread + return thread + wrapper.__name__ = func.__name__ + + return wrapper + + # Cleanup finished threads + def cleanup(self, key, thread): + if key in self.threads: + del(self.threads[key]) + + +if __name__ == "__main__": + + + class Test(): + + @Noparallel() + def count(self, num=5): + for i in range(num): + print(self, i) + time.sleep(1) + return "%s return:%s" % (self, i) + + class TestNoblock(): + + @Noparallel(blocking=False) + def count(self, num=5): + for i in range(num): + print(self, i) + time.sleep(1) + return "%s return:%s" % (self, i) + + def testBlocking(): + test = Test() + test2 = Test() + print("Counting...") + print("Creating class1/thread1") + thread1 = gevent.spawn(test.count) + print("Creating class1/thread2 (ignored)") + thread2 = gevent.spawn(test.count) + print("Creating class2/thread3") + thread3 = gevent.spawn(test2.count) + + print("Joining class1/thread1") + thread1.join() + print("Joining class1/thread2") + thread2.join() + print("Joining class2/thread3") + thread3.join() + + print("Creating class1/thread4 (its finished, allowed again)") + thread4 = gevent.spawn(test.count) + print("Joining thread4") + thread4.join() + + print(thread1.value, thread2.value, thread3.value, thread4.value) + print("Done.") + + def testNoblocking(): + test = TestNoblock() + test2 = TestNoblock() + print("Creating class1/thread1") + thread1 = test.count() + print("Creating class1/thread2 (ignored)") + thread2 = test.count() + print("Creating class2/thread3") + thread3 = test2.count() + print("Joining class1/thread1") + thread1.join() + print("Joining class1/thread2") + thread2.join() + print("Joining class2/thread3") + thread3.join() + + print("Creating class1/thread4 (its finished, allowed again)") + thread4 = test.count() + print("Joining thread4") + thread4.join() + + print(thread1.value, thread2.value, thread3.value, thread4.value) + print("Done.") + + def testBenchmark(): + import time + + def printThreadNum(): + import gc + from greenlet import greenlet + objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)] + print("Greenlets: %s" % len(objs)) + + printThreadNum() + test = TestNoblock() + s = time.time() + for i in range(3): + gevent.spawn(test.count, i + 1) + print("Created in %.3fs" % (time.time() - s)) + printThreadNum() + time.sleep(5) + + def testException(): + import time + @Noparallel(blocking=True, queue=True) + def count(self, num=5): + s = time.time() + # raise Exception("err") + for i in range(num): + print(self, i) + time.sleep(1) + return "%s return:%s" % (s, i) + def caller(): + try: + print("Ret:", count(5)) + except Exception as err: + print("Raised:", repr(err)) + + gevent.joinall([ + gevent.spawn(caller), + gevent.spawn(caller), + gevent.spawn(caller), + gevent.spawn(caller) + ]) + + + from gevent import monkey + monkey.patch_all() + + testException() + + """ + testBenchmark() + print("Testing blocking mode...") + testBlocking() + print("Testing noblocking mode...") + testNoblocking() + """ diff --git a/src/util/OpensslFindPatch.py b/src/util/OpensslFindPatch.py new file mode 100644 index 00000000..0f5d2dc6 --- /dev/null +++ b/src/util/OpensslFindPatch.py @@ -0,0 +1,69 @@ +import logging +import os +import sys +import ctypes.util + +from Config import config + +find_library_original = ctypes.util.find_library + + +def getOpensslPath(): + if config.openssl_lib_file: + return config.openssl_lib_file + + if sys.platform.startswith("win"): + lib_paths = [ + os.path.join(os.getcwd(), "tools/openssl/libeay32.dll"), # ZeroBundle Windows + os.path.join(os.path.dirname(sys.executable), "DLLs/libcrypto-1_1-x64.dll"), + os.path.join(os.path.dirname(sys.executable), "DLLs/libcrypto-1_1.dll") + ] + elif sys.platform == "cygwin": + lib_paths = ["/bin/cygcrypto-1.0.0.dll"] + else: + lib_paths = [ + "../runtime/lib/libcrypto.so.1.1", # ZeroBundle Linux + "../../Frameworks/libcrypto.1.1.dylib", # ZeroBundle macOS + "/opt/lib/libcrypto.so.1.0.0", # For optware and entware + "/usr/local/ssl/lib/libcrypto.so" + ] + + for lib_path in lib_paths: + if os.path.isfile(lib_path): + return lib_path + + if "ANDROID_APP_PATH" in os.environ: + try: + lib_dir = os.environ["ANDROID_APP_PATH"] + "/../../lib" + return [lib for lib in os.listdir(lib_dir) if "crypto" in lib][0] + except Exception as err: + logging.debug("OpenSSL lib not found in: %s (%s)" % (lib_dir, err)) + + if "LD_LIBRARY_PATH" in os.environ: + lib_dir_paths = os.environ["LD_LIBRARY_PATH"].split(":") + for path in lib_dir_paths: + try: + return [lib for lib in os.listdir(path) if "libcrypto.so" in lib][0] + except Exception as err: + logging.debug("OpenSSL lib not found in: %s (%s)" % (path, err)) + + lib_path = ( + find_library_original('ssl.so') or find_library_original('ssl') or + find_library_original('crypto') or find_library_original('libcrypto') or 'libeay32' + ) + + return lib_path + + +def patchCtypesOpensslFindLibrary(): + def findLibraryPatched(name): + if name in ("ssl", "crypto", "libeay32"): + lib_path = getOpensslPath() + return lib_path + else: + return find_library_original(name) + + ctypes.util.find_library = findLibraryPatched + + +patchCtypesOpensslFindLibrary() diff --git a/src/util/Platform.py b/src/util/Platform.py new file mode 100644 index 00000000..5bdde2f8 --- /dev/null +++ b/src/util/Platform.py @@ -0,0 +1,36 @@ +import sys +import logging + + +def setMaxfilesopened(limit): + try: + if sys.platform == "win32": + import ctypes + dll = None + last_err = None + for dll_name in ["msvcr100", "msvcr110", "msvcr120"]: + try: + dll = getattr(ctypes.cdll, dll_name) + break + except OSError as err: + last_err = err + + if not dll: + raise last_err + + maxstdio = dll._getmaxstdio() + if maxstdio < limit: + logging.debug("%s: Current maxstdio: %s, changing to %s..." % (dll, maxstdio, limit)) + dll._setmaxstdio(limit) + return True + else: + import resource + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < limit: + logging.debug("Current RLIMIT_NOFILE: %s (max: %s), changing to %s..." % (soft, hard, limit)) + resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) + return True + + except Exception as err: + logging.error("Failed to modify max files open limit: %s" % err) + return False diff --git a/src/util/Pooled.py b/src/util/Pooled.py new file mode 100644 index 00000000..9a4a7b63 --- /dev/null +++ b/src/util/Pooled.py @@ -0,0 +1,65 @@ +import gevent.pool + + +class Pooled(object): + def __init__(self, size=100): + self.pool = gevent.pool.Pool(size) + self.pooler_running = False + self.queue = [] + self.func = None + + def waiter(self, evt, args, kwargs): + res = self.func(*args, **kwargs) + if type(res) == gevent.event.AsyncResult: + evt.set(res.get()) + else: + evt.set(res) + + def pooler(self): + while self.queue: + evt, args, kwargs = self.queue.pop(0) + self.pool.spawn(self.waiter, evt, args, kwargs) + self.pooler_running = False + + def __call__(self, func): + def wrapper(*args, **kwargs): + evt = gevent.event.AsyncResult() + self.queue.append((evt, args, kwargs)) + if not self.pooler_running: + self.pooler_running = True + gevent.spawn(self.pooler) + return evt + wrapper.__name__ = func.__name__ + self.func = func + + return wrapper + +if __name__ == "__main__": + import gevent + import gevent.pool + import gevent.queue + import gevent.event + import gevent.monkey + import time + + gevent.monkey.patch_all() + + def addTask(inner_path): + evt = gevent.event.AsyncResult() + gevent.spawn_later(1, lambda: evt.set(True)) + return evt + + def needFile(inner_path): + return addTask(inner_path) + + @Pooled(10) + def pooledNeedFile(inner_path): + return needFile(inner_path) + + threads = [] + for i in range(100): + threads.append(pooledNeedFile(i)) + + s = time.time() + gevent.joinall(threads) # Should take 10 second + print(time.time() - s) diff --git a/src/util/QueryJson.py b/src/util/QueryJson.py new file mode 100644 index 00000000..d9921ff0 --- /dev/null +++ b/src/util/QueryJson.py @@ -0,0 +1,67 @@ +import json +import re +import os + + +def queryFile(file_path, filter_path, filter_key=None, filter_val=None): + back = [] + data = json.load(open(file_path)) + if filter_path == ['']: + return [data] + for key in filter_path: # Get to the point + data = data.get(key) + if not data: + return + + if type(data) == list: + for row in data: + if filter_val: # Filter by value + if row[filter_key] == filter_val: + back.append(row) + else: + back.append(row) + else: + back.append({"value": data}) + + return back + + +# Find in json files +# Return: [{u'body': u'Hello Topic 1!!', 'inner_path': '1KRxE1...beEp6', u'added': 1422740732, u'message_id': 1},...] +def query(path_pattern, filter): + if "=" in filter: # Filter by value + filter_path, filter_val = filter.split("=") + filter_path = filter_path.split(".") + filter_key = filter_path.pop() # Last element is the key + filter_val = int(filter_val) + else: # No filter + filter_path = filter + filter_path = filter_path.split(".") + filter_key = None + filter_val = None + + if "/*/" in path_pattern: # Wildcard search + root_dir, file_pattern = path_pattern.replace("\\", "/").split("/*/") + else: # No wildcard + root_dir, file_pattern = re.match("(.*)/(.*?)$", path_pattern.replace("\\", "/")).groups() + for root, dirs, files in os.walk(root_dir, topdown=False): + root = root.replace("\\", "/") + inner_path = root.replace(root_dir, "").strip("/") + for file_name in files: + if file_pattern != file_name: + continue + + try: + res = queryFile(root + "/" + file_name, filter_path, filter_key, filter_val) + if not res: + continue + except Exception: # Json load error + continue + for row in res: + row["inner_path"] = inner_path + yield row + + +if __name__ == "__main__": + for row in list(query("../../data/12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH/data/users/*/data.json", "")): + print(row) diff --git a/src/util/RateLimit.py b/src/util/RateLimit.py new file mode 100644 index 00000000..465859c2 --- /dev/null +++ b/src/util/RateLimit.py @@ -0,0 +1,128 @@ +import time +import gevent +import logging + +log = logging.getLogger("RateLimit") + +called_db = {} # Holds events last call time +queue_db = {} # Commands queued to run + +# Register event as called +# Return: None + + +def called(event, penalty=0): + called_db[event] = time.time() + penalty + + +# Check if calling event is allowed +# Return: True if allowed False if not +def isAllowed(event, allowed_again=10): + last_called = called_db.get(event) + if not last_called: # Its not called before + return True + elif time.time() - last_called >= allowed_again: + del called_db[event] # Delete last call time to save memory + return True + else: + return False + +def delayLeft(event, allowed_again=10): + last_called = called_db.get(event) + if not last_called: # Its not called before + return 0 + else: + return allowed_again - (time.time() - last_called) + +def callQueue(event): + func, args, kwargs, thread = queue_db[event] + log.debug("Calling: %s" % event) + called(event) + del queue_db[event] + return func(*args, **kwargs) + + +# Rate limit and delay function call if necessary +# If the function called again within the rate limit interval then previous queued call will be dropped +# Return: Immediately gevent thread +def callAsync(event, allowed_again=10, func=None, *args, **kwargs): + if isAllowed(event, allowed_again): # Not called recently, call it now + called(event) + # print "Calling now" + return gevent.spawn(func, *args, **kwargs) + else: # Called recently, schedule it for later + time_left = allowed_again - max(0, time.time() - called_db[event]) + log.debug("Added to queue (%.2fs left): %s " % (time_left, event)) + if not queue_db.get(event): # Function call not queued yet + thread = gevent.spawn_later(time_left, lambda: callQueue(event)) # Call this function later + queue_db[event] = (func, args, kwargs, thread) + return thread + else: # Function call already queued, just update the parameters + thread = queue_db[event][3] + queue_db[event] = (func, args, kwargs, thread) + return thread + + +# Rate limit and delay function call if needed +# Return: Wait for execution/delay then return value +def call(event, allowed_again=10, func=None, *args, **kwargs): + if isAllowed(event): # Not called recently, call it now + called(event) + # print "Calling now", allowed_again + return func(*args, **kwargs) + + else: # Called recently, schedule it for later + time_left = max(0, allowed_again - (time.time() - called_db[event])) + # print "Time left: %s" % time_left, args, kwargs + log.debug("Calling sync (%.2fs left): %s" % (time_left, event)) + called(event, time_left) + time.sleep(time_left) + back = func(*args, **kwargs) + called(event) + return back + + +# Cleanup expired events every 3 minutes +def rateLimitCleanup(): + while 1: + expired = time.time() - 60 * 2 # Cleanup if older than 2 minutes + for event in list(called_db.keys()): + if called_db[event] < expired: + del called_db[event] + time.sleep(60 * 3) # Every 3 minutes +gevent.spawn(rateLimitCleanup) + + +if __name__ == "__main__": + from gevent import monkey + monkey.patch_all() + import random + + def publish(inner_path): + print("Publishing %s..." % inner_path) + return 1 + + def cb(thread): + print("Value:", thread.value) + + print("Testing async spam requests rate limit to 1/sec...") + for i in range(3000): + thread = callAsync("publish content.json", 1, publish, "content.json %s" % i) + time.sleep(float(random.randint(1, 20)) / 100000) + print(thread.link(cb)) + print("Done") + + time.sleep(2) + + print("Testing sync spam requests rate limit to 1/sec...") + for i in range(5): + call("publish data.json", 1, publish, "data.json %s" % i) + time.sleep(float(random.randint(1, 100)) / 100) + print("Done") + + print("Testing cleanup") + thread = callAsync("publish content.json single", 1, publish, "content.json single") + print("Needs to cleanup:", called_db, queue_db) + print("Waiting 3min for cleanup process...") + time.sleep(60 * 3) + print("Cleaned up:", called_db, queue_db) diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py new file mode 100644 index 00000000..6018e2d3 --- /dev/null +++ b/src/util/SafeRe.py @@ -0,0 +1,32 @@ +import re + + +class UnsafePatternError(Exception): + pass + +cached_patterns = {} + + +def isSafePattern(pattern): + if len(pattern) > 255: + raise UnsafePatternError("Pattern too long: %s characters in %s" % (len(pattern), pattern)) + + unsafe_pattern_match = re.search(r"[^\.][\*\{\+]", pattern) # Always should be "." before "*{+" characters to avoid ReDoS + if unsafe_pattern_match: + raise UnsafePatternError("Potentially unsafe part of the pattern: %s in %s" % (unsafe_pattern_match.group(0), pattern)) + + repetitions = re.findall(r"\.[\*\{\+]", pattern) + if len(repetitions) >= 10: + raise UnsafePatternError("More than 10 repetitions of %s in %s" % (repetitions[0], pattern)) + + return True + + +def match(pattern, *args, **kwargs): + cached_pattern = cached_patterns.get(pattern) + if cached_pattern: + return cached_pattern.match(*args, **kwargs) + else: + if isSafePattern(pattern): + cached_patterns[pattern] = re.compile(pattern) + return cached_patterns[pattern].match(*args, **kwargs) diff --git a/src/util/SocksProxy.py b/src/util/SocksProxy.py new file mode 100644 index 00000000..f831137b --- /dev/null +++ b/src/util/SocksProxy.py @@ -0,0 +1,26 @@ +import socket + +import socks +from Config import config + +def create_connection(address, timeout=None, source_address=None): + if address in config.ip_local: + sock = socket.socket_noproxy(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(address) + else: + sock = socks.socksocket() + sock.connect(address) + return sock + + +# Dns queries using the proxy +def getaddrinfo(*args): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] + + +def monkeyPatch(proxy_ip, proxy_port): + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, proxy_ip, int(proxy_port)) + socket.socket_noproxy = socket.socket + socket.socket = socks.socksocket + socket.create_connection = create_connection + socket.getaddrinfo = getaddrinfo diff --git a/src/util/ThreadPool.py b/src/util/ThreadPool.py new file mode 100644 index 00000000..5b31ce37 --- /dev/null +++ b/src/util/ThreadPool.py @@ -0,0 +1,180 @@ +import threading +import time +import queue + +import gevent +import gevent.monkey +import gevent.threadpool +import gevent._threading + + +class ThreadPool: + def __init__(self, max_size, name=None): + self.setMaxSize(max_size) + if name: + self.name = name + else: + self.name = "ThreadPool#%s" % id(self) + + def setMaxSize(self, max_size): + self.max_size = max_size + if max_size > 0: + self.pool = gevent.threadpool.ThreadPool(max_size) + else: + self.pool = None + + def wrap(self, func): + if self.pool is None: + return func + + def wrapper(*args, **kwargs): + if not isMainThread(): # Call directly if not in main thread + return func(*args, **kwargs) + res = self.apply(func, args, kwargs) + return res + + return wrapper + + def spawn(self, *args, **kwargs): + if not isMainThread() and not self.pool._semaphore.ready(): + # Avoid semaphore error when spawning from other thread and the pool is full + return main_loop.call(self.spawn, *args, **kwargs) + res = self.pool.spawn(*args, **kwargs) + return res + + def apply(self, func, args=(), kwargs={}): + t = self.spawn(func, *args, **kwargs) + if self.pool._apply_immediately(): + return main_loop.call(t.get) + else: + return t.get() + + def kill(self): + if self.pool is not None and self.pool.size > 0 and main_loop: + main_loop.call(lambda: gevent.spawn(self.pool.kill).join(timeout=1)) + + del self.pool + self.pool = None + + def __enter__(self): + return self + + def __exit__(self, *args): + self.kill() + + +lock_pool = gevent.threadpool.ThreadPool(50) +main_thread_id = threading.current_thread().ident + + +def isMainThread(): + return threading.current_thread().ident == main_thread_id + + +class Lock: + def __init__(self): + self.lock = gevent._threading.Lock() + self.locked = self.lock.locked + self.release = self.lock.release + self.time_lock = 0 + + def acquire(self, *args, **kwargs): + self.time_lock = time.time() + if self.locked() and isMainThread(): + # Start in new thread to avoid blocking gevent loop + return lock_pool.apply(self.lock.acquire, args, kwargs) + else: + return self.lock.acquire(*args, **kwargs) + + def __del__(self): + while self.locked(): + self.release() + + +class Event: + def __init__(self): + self.get_lock = Lock() + self.res = None + self.get_lock.acquire(False) + self.done = False + + def set(self, res): + if self.done: + raise Exception("Event already has value") + self.res = res + self.get_lock.release() + self.done = True + + def get(self): + if not self.done: + self.get_lock.acquire(True) + if self.get_lock.locked(): + self.get_lock.release() + back = self.res + return back + + def __del__(self): + self.res = None + while self.get_lock.locked(): + self.get_lock.release() + + +# Execute function calls in main loop from other threads +class MainLoopCaller(): + def __init__(self): + self.queue_call = queue.Queue() + + self.pool = gevent.threadpool.ThreadPool(1) + self.num_direct = 0 + self.running = True + + def caller(self, func, args, kwargs, event_done): + try: + res = func(*args, **kwargs) + event_done.set((True, res)) + except Exception as err: + event_done.set((False, err)) + + def start(self): + gevent.spawn(self.run) + time.sleep(0.001) + + def run(self): + while self.running: + if self.queue_call.qsize() == 0: # Get queue in new thread to avoid gevent blocking + func, args, kwargs, event_done = self.pool.apply(self.queue_call.get) + else: + func, args, kwargs, event_done = self.queue_call.get() + gevent.spawn(self.caller, func, args, kwargs, event_done) + del func, args, kwargs, event_done + self.running = False + + def call(self, func, *args, **kwargs): + if threading.current_thread().ident == main_thread_id: + return func(*args, **kwargs) + else: + event_done = Event() + self.queue_call.put((func, args, kwargs, event_done)) + success, res = event_done.get() + del event_done + self.queue_call.task_done() + if success: + return res + else: + raise res + + +def patchSleep(): # Fix memory leak by using real sleep in threads + real_sleep = gevent.monkey.get_original("time", "sleep") + + def patched_sleep(seconds): + if isMainThread(): + gevent.sleep(seconds) + else: + real_sleep(seconds) + time.sleep = patched_sleep + + +main_loop = MainLoopCaller() +main_loop.start() +patchSleep() diff --git a/src/util/UpnpPunch.py b/src/util/UpnpPunch.py new file mode 100644 index 00000000..18f4aaee --- /dev/null +++ b/src/util/UpnpPunch.py @@ -0,0 +1,395 @@ +import re +import urllib.request +import http.client +import logging +from urllib.parse import urlparse +from xml.dom.minidom import parseString +from xml.parsers.expat import ExpatError + +from gevent import socket +import gevent + +# Relevant UPnP spec: +# http://www.upnp.org/specs/gw/UPnP-gw-WANIPConnection-v1-Service.pdf + +# General TODOs: +# Handle 0 or >1 IGDs + +logger = logging.getLogger("Upnp") + +class UpnpError(Exception): + pass + + +class IGDError(UpnpError): + """ + Signifies a problem with the IGD. + """ + pass + + +REMOVE_WHITESPACE = re.compile(r'>\s*<') + + +def perform_m_search(local_ip): + """ + Broadcast a UDP SSDP M-SEARCH packet and return response. + """ + search_target = "urn:schemas-upnp-org:device:InternetGatewayDevice:1" + + ssdp_request = ''.join( + ['M-SEARCH * HTTP/1.1\r\n', + 'HOST: 239.255.255.250:1900\r\n', + 'MAN: "ssdp:discover"\r\n', + 'MX: 2\r\n', + 'ST: {0}\r\n'.format(search_target), + '\r\n'] + ).encode("utf8") + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + sock.bind((local_ip, 0)) + + sock.sendto(ssdp_request, ('239.255.255.250', 1900)) + if local_ip == "127.0.0.1": + sock.settimeout(1) + else: + sock.settimeout(5) + + try: + return sock.recv(2048).decode("utf8") + except socket.error: + raise UpnpError("No reply from IGD using {} as IP".format(local_ip)) + finally: + sock.close() + + +def _retrieve_location_from_ssdp(response): + """ + Parse raw HTTP response to retrieve the UPnP location header + and return a ParseResult object. + """ + parsed_headers = re.findall(r'(?P.*?): (?P.*?)\r\n', response) + header_locations = [header[1] + for header in parsed_headers + if header[0].lower() == 'location'] + + if len(header_locations) < 1: + raise IGDError('IGD response does not contain a "location" header.') + + return urlparse(header_locations[0]) + + +def _retrieve_igd_profile(url): + """ + Retrieve the device's UPnP profile. + """ + try: + return urllib.request.urlopen(url.geturl(), timeout=5).read().decode('utf-8') + except socket.error: + raise IGDError('IGD profile query timed out') + + +def _get_first_child_data(node): + """ + Get the text value of the first child text node of a node. + """ + return node.childNodes[0].data + + +def _parse_igd_profile(profile_xml): + """ + Traverse the profile xml DOM looking for either + WANIPConnection or WANPPPConnection and return + the 'controlURL' and the service xml schema. + """ + try: + dom = parseString(profile_xml) + except ExpatError as e: + raise IGDError( + 'Unable to parse IGD reply: {0} \n\n\n {1}'.format(profile_xml, e)) + + service_types = dom.getElementsByTagName('serviceType') + for service in service_types: + if _get_first_child_data(service).find('WANIPConnection') > 0 or \ + _get_first_child_data(service).find('WANPPPConnection') > 0: + try: + control_url = _get_first_child_data( + service.parentNode.getElementsByTagName('controlURL')[0]) + upnp_schema = _get_first_child_data(service).split(':')[-2] + return control_url, upnp_schema + except IndexError: + # Pass the error because any error here should raise the + # that's specified outside the for loop. + pass + raise IGDError( + 'Could not find a control url or UPNP schema in IGD response.') + + +# add description +def _get_local_ips(): + def method1(): + try: + # get local ip using UDP and a broadcast address + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + # Not using because gevents getaddrinfo doesn't like that + # using port 1 as per hobbldygoop's comment about port 0 not working on osx: + # https://github.com/sirMackk/ZeroNet/commit/fdcd15cf8df0008a2070647d4d28ffedb503fba2#commitcomment-9863928 + s.connect(('239.255.255.250', 1)) + return [s.getsockname()[0]] + except: + pass + + def method2(): + # Get ip by using UDP and a normal address (google dns ip) + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(('8.8.8.8', 0)) + return [s.getsockname()[0]] + except: + pass + + def method3(): + # Get ip by '' hostname . Not supported on all platforms. + try: + return socket.gethostbyname_ex('')[2] + except: + pass + + threads = [ + gevent.spawn(method1), + gevent.spawn(method2), + gevent.spawn(method3) + ] + + gevent.joinall(threads, timeout=5) + + local_ips = [] + for thread in threads: + if thread.value: + local_ips += thread.value + + # Delete duplicates + local_ips = list(set(local_ips)) + + + # Probably we looking for an ip starting with 192 + local_ips = sorted(local_ips, key=lambda a: a.startswith("192"), reverse=True) + + return local_ips + + +def _create_open_message(local_ip, + port, + description="UPnPPunch", + protocol="TCP", + upnp_schema='WANIPConnection'): + """ + Build a SOAP AddPortMapping message. + """ + + soap_message = """ + + + + + {port} + {protocol} + {port} + {host_ip} + 1 + {description} + 0 + + +""".format(port=port, + protocol=protocol, + host_ip=local_ip, + description=description, + upnp_schema=upnp_schema) + return (REMOVE_WHITESPACE.sub('><', soap_message), 'AddPortMapping') + + +def _create_close_message(local_ip, + port, + description=None, + protocol='TCP', + upnp_schema='WANIPConnection'): + soap_message = """ + + + + + {port} + {protocol} + + +""".format(port=port, + protocol=protocol, + upnp_schema=upnp_schema) + return (REMOVE_WHITESPACE.sub('><', soap_message), 'DeletePortMapping') + + +def _parse_for_errors(soap_response): + logger.debug(soap_response.status) + if soap_response.status >= 400: + response_data = soap_response.read() + logger.debug(response_data) + try: + err_dom = parseString(response_data) + err_code = _get_first_child_data(err_dom.getElementsByTagName( + 'errorCode')[0]) + err_msg = _get_first_child_data( + err_dom.getElementsByTagName('errorDescription')[0] + ) + except Exception as err: + raise IGDError( + 'Unable to parse SOAP error: {0}. Got: "{1}"'.format( + err, response_data)) + raise IGDError( + 'SOAP request error: {0} - {1}'.format(err_code, err_msg) + ) + return soap_response + + +def _send_soap_request(location, upnp_schema, control_path, soap_fn, + soap_message): + """ + Send out SOAP request to UPnP device and return a response. + """ + headers = { + 'SOAPAction': ( + '"urn:schemas-upnp-org:service:{schema}:' + '1#{fn_name}"'.format(schema=upnp_schema, fn_name=soap_fn) + ), + 'Content-Type': 'text/xml' + } + logger.debug("Sending UPnP request to {0}:{1}...".format( + location.hostname, location.port)) + conn = http.client.HTTPConnection(location.hostname, location.port) + conn.request('POST', control_path, soap_message, headers) + + response = conn.getresponse() + conn.close() + + return _parse_for_errors(response) + + +def _collect_idg_data(ip_addr): + idg_data = {} + idg_response = perform_m_search(ip_addr) + idg_data['location'] = _retrieve_location_from_ssdp(idg_response) + idg_data['control_path'], idg_data['upnp_schema'] = _parse_igd_profile( + _retrieve_igd_profile(idg_data['location'])) + return idg_data + + +def _send_requests(messages, location, upnp_schema, control_path): + responses = [_send_soap_request(location, upnp_schema, control_path, + message_tup[1], message_tup[0]) + for message_tup in messages] + + if all(rsp.status == 200 for rsp in responses): + return + raise UpnpError('Sending requests using UPnP failed.') + + +def _orchestrate_soap_request(ip, port, msg_fn, desc=None, protos=("TCP", "UDP")): + logger.debug("Trying using local ip: %s" % ip) + idg_data = _collect_idg_data(ip) + + soap_messages = [ + msg_fn(ip, port, desc, proto, idg_data['upnp_schema']) + for proto in protos + ] + + _send_requests(soap_messages, **idg_data) + + +def _communicate_with_igd(port=15441, + desc="UpnpPunch", + retries=3, + fn=_create_open_message, + protos=("TCP", "UDP")): + """ + Manage sending a message generated by 'fn'. + """ + + local_ips = _get_local_ips() + success = False + + def job(local_ip): + for retry in range(retries): + try: + _orchestrate_soap_request(local_ip, port, fn, desc, protos) + return True + except Exception as e: + logger.debug('Upnp request using "{0}" failed: {1}'.format(local_ip, e)) + gevent.sleep(1) + return False + + threads = [] + + for local_ip in local_ips: + job_thread = gevent.spawn(job, local_ip) + threads.append(job_thread) + gevent.sleep(0.1) + if any([thread.value for thread in threads]): + success = True + break + + # Wait another 10sec for competition or any positive result + for _ in range(10): + all_done = all([thread.value is not None for thread in threads]) + any_succeed = any([thread.value for thread in threads]) + if all_done or any_succeed: + break + gevent.sleep(1) + + if any([thread.value for thread in threads]): + success = True + + if not success: + raise UpnpError( + 'Failed to communicate with igd using port {0} on local machine after {1} tries.'.format( + port, retries)) + + return success + + +def ask_to_open_port(port=15441, desc="UpnpPunch", retries=3, protos=("TCP", "UDP")): + logger.debug("Trying to open port %d." % port) + return _communicate_with_igd(port=port, + desc=desc, + retries=retries, + fn=_create_open_message, + protos=protos) + + +def ask_to_close_port(port=15441, desc="UpnpPunch", retries=3, protos=("TCP", "UDP")): + logger.debug("Trying to close port %d." % port) + # retries=1 because multiple successes cause 500 response and failure + return _communicate_with_igd(port=port, + desc=desc, + retries=retries, + fn=_create_close_message, + protos=protos) + + +if __name__ == "__main__": + from gevent import monkey + monkey.patch_all() + logging.basicConfig(level=logging.DEBUG) + import time + + s = time.time() + print("Opening port...") + print("Success:", ask_to_open_port(15443, "ZeroNet", protos=["TCP"])) + print("Done in", time.time() - s) + + + print("Closing port...") + print("Success:", ask_to_close_port(15443, "ZeroNet", protos=["TCP"])) + print("Done in", time.time() - s) + diff --git a/src/util/__init__.py b/src/util/__init__.py new file mode 100644 index 00000000..ab8a8b88 --- /dev/null +++ b/src/util/__init__.py @@ -0,0 +1,4 @@ +from .Cached import Cached +from .Event import Event +from .Noparallel import Noparallel +from .Pooled import Pooled diff --git a/src/util/helper.py b/src/util/helper.py new file mode 100644 index 00000000..61455b08 --- /dev/null +++ b/src/util/helper.py @@ -0,0 +1,356 @@ +import os +import stat +import socket +import struct +import re +import collections +import time +import logging +import base64 +import json + +import gevent + +from Config import config + + +def atomicWrite(dest, content, mode="wb"): + try: + with open(dest + "-tmpnew", mode) as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + if os.path.isfile(dest + "-tmpold"): # Previous incomplete write + os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time()) + if os.path.isfile(dest): # Rename old file to -tmpold + os.rename(dest, dest + "-tmpold") + os.rename(dest + "-tmpnew", dest) + if os.path.isfile(dest + "-tmpold"): + os.unlink(dest + "-tmpold") # Remove old file + return True + except Exception as err: + from Debug import Debug + logging.error( + "File %s write failed: %s, (%s) reverting..." % + (dest, Debug.formatException(err), Debug.formatStack()) + ) + if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest): + os.rename(dest + "-tmpold", dest) + return False + + +def jsonDumps(data): + content = json.dumps(data, indent=1, sort_keys=True) + + # Make it a little more compact by removing unnecessary white space + def compact_dict(match): + if "\n" in match.group(0): + return match.group(0).replace(match.group(1), match.group(1).strip()) + else: + return match.group(0) + + content = re.sub(r"\{(\n[^,\[\{]{10,100000}?)\}[, ]{0,2}\n", compact_dict, content, flags=re.DOTALL) + + def compact_list(match): + if "\n" in match.group(0): + stripped_lines = re.sub("\n[ ]*", "", match.group(1)) + return match.group(0).replace(match.group(1), stripped_lines) + else: + return match.group(0) + + content = re.sub(r"\[([^\[\{]{2,100000}?)\][, ]{0,2}\n", compact_list, content, flags=re.DOTALL) + + # Remove end of line whitespace + content = re.sub(r"(?m)[ ]+$", "", content) + return content + + +def openLocked(path, mode="wb"): + try: + if os.name == "posix": + import fcntl + f = open(path, mode) + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + elif os.name == "nt": + import msvcrt + f = open(path, mode) + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + f = open(path, mode) + except (IOError, PermissionError, BlockingIOError) as err: + raise BlockingIOError("Unable to lock file: %s" % err) + return f + + +def getFreeSpace(): + free_space = -1 + if "statvfs" in dir(os): # Unix + statvfs = os.statvfs(config.data_dir.encode("utf8")) + free_space = statvfs.f_frsize * statvfs.f_bavail + else: # Windows + try: + import ctypes + free_space_pointer = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW( + ctypes.c_wchar_p(config.data_dir), None, None, ctypes.pointer(free_space_pointer) + ) + free_space = free_space_pointer.value + except Exception as err: + logging.error("GetFreeSpace error: %s" % err) + return free_space + + +def sqlquote(value): + if type(value) is int: + return str(value) + else: + return "'%s'" % value.replace("'", "''") + + +def shellquote(*args): + if len(args) == 1: + return '"%s"' % args[0].replace('"', "") + else: + return tuple(['"%s"' % arg.replace('"', "") for arg in args]) + + +def packPeers(peers): + packed_peers = {"ipv4": [], "ipv6": [], "onion": []} + for peer in peers: + try: + ip_type = getIpType(peer.ip) + if ip_type in packed_peers: + packed_peers[ip_type].append(peer.packMyAddress()) + except Exception: + logging.debug("Error packing peer address: %s" % peer) + return packed_peers + + +# ip, port to packed 6byte or 18byte format +def packAddress(ip, port): + if ":" in ip: + return socket.inet_pton(socket.AF_INET6, ip) + struct.pack("H", port) + else: + return socket.inet_aton(ip) + struct.pack("H", port) + + +# From 6byte or 18byte format to ip, port +def unpackAddress(packed): + if len(packed) == 18: + return socket.inet_ntop(socket.AF_INET6, packed[0:16]), struct.unpack_from("H", packed, 16)[0] + else: + if len(packed) != 6: + raise Exception("Invalid length ip4 packed address: %s" % len(packed)) + return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0] + + +# onion, port to packed 12byte format +def packOnionAddress(onion, port): + onion = onion.replace(".onion", "") + return base64.b32decode(onion.upper()) + struct.pack("H", port) + + +# From 12byte format to ip, port +def unpackOnionAddress(packed): + return base64.b32encode(packed[0:-2]).lower().decode() + ".onion", struct.unpack("H", packed[-2:])[0] + + +# Get dir from file +# Return: data/site/content.json -> data/site/ +def getDirname(path): + if "/" in path: + return path[:path.rfind("/") + 1].lstrip("/") + else: + return "" + + +# Get dir from file +# Return: data/site/content.json -> content.json +def getFilename(path): + return path[path.rfind("/") + 1:] + + +def getFilesize(path): + try: + s = os.stat(path) + except Exception: + return None + if stat.S_ISREG(s.st_mode): # Test if it's file + return s.st_size + else: + return None + + +# Convert hash to hashid for hashfield +def toHashId(hash): + return int(hash[0:4], 16) + + +# Merge dict values +def mergeDicts(dicts): + back = collections.defaultdict(set) + for d in dicts: + for key, val in d.items(): + back[key].update(val) + return dict(back) + + +# Request https url using gevent SSL error workaround +def httpRequest(url, as_file=False): + if url.startswith("http://"): + import urllib.request + response = urllib.request.urlopen(url) + else: # Hack to avoid Python gevent ssl errors + import socket + import http.client + import ssl + + host, request = re.match("https://(.*?)(/.*?)$", url).groups() + + conn = http.client.HTTPSConnection(host) + sock = socket.create_connection((conn.host, conn.port), conn.timeout, conn.source_address) + conn.sock = ssl.wrap_socket(sock, conn.key_file, conn.cert_file) + conn.request("GET", request) + response = conn.getresponse() + if response.status in [301, 302, 303, 307, 308]: + logging.info("Redirect to: %s" % response.getheader('Location')) + response = httpRequest(response.getheader('Location')) + + if as_file: + import io + data = io.BytesIO() + while True: + buff = response.read(1024 * 16) + if not buff: + break + data.write(buff) + return data + else: + return response + + +def timerCaller(secs, func, *args, **kwargs): + gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs) + func(*args, **kwargs) + + +def timer(secs, func, *args, **kwargs): + return gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs) + + +def create_connection(address, timeout=None, source_address=None): + if address in config.ip_local: + sock = socket.create_connection_original(address, timeout, source_address) + else: + sock = socket.create_connection_original(address, timeout, socket.bind_addr) + return sock + + +def socketBindMonkeyPatch(bind_ip, bind_port): + import socket + logging.info("Monkey patching socket to bind to: %s:%s" % (bind_ip, bind_port)) + socket.bind_addr = (bind_ip, int(bind_port)) + socket.create_connection_original = socket.create_connection + socket.create_connection = create_connection + + +def limitedGzipFile(*args, **kwargs): + import gzip + + class LimitedGzipFile(gzip.GzipFile): + def read(self, size=-1): + return super(LimitedGzipFile, self).read(1024 * 1024 * 25) + return LimitedGzipFile(*args, **kwargs) + + +def avg(items): + if len(items) > 0: + return sum(items) / len(items) + else: + return 0 + + +def isIp(ip): + if ":" in ip: # IPv6 + try: + socket.inet_pton(socket.AF_INET6, ip) + return True + except Exception: + return False + + else: # IPv4 + try: + socket.inet_aton(ip) + return True + except Exception: + return False + + +local_ip_pattern = re.compile(r"^127\.|192\.168\.|10\.|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[0-1]\.|169\.254\.|::1$|fe80") +def isPrivateIp(ip): + return local_ip_pattern.match(ip) + + +def getIpType(ip): + if ip.endswith(".onion"): + return "onion" + elif ":" in ip: + return "ipv6" + elif re.match(r"[0-9\.]+$", ip): + return "ipv4" + else: + return "unknown" + + +def createSocket(ip, sock_type=socket.SOCK_STREAM): + ip_type = getIpType(ip) + if ip_type == "ipv6": + return socket.socket(socket.AF_INET6, sock_type) + else: + return socket.socket(socket.AF_INET, sock_type) + + +def getInterfaceIps(ip_type="ipv4"): + res = [] + if ip_type == "ipv6": + test_ips = ["ff0e::c", "2606:4700:4700::1111"] + else: + test_ips = ['239.255.255.250', "8.8.8.8"] + + for test_ip in test_ips: + try: + s = createSocket(test_ip, sock_type=socket.SOCK_DGRAM) + s.connect((test_ip, 1)) + res.append(s.getsockname()[0]) + except Exception: + pass + + try: + res += [ip[4][0] for ip in socket.getaddrinfo(socket.gethostname(), 1)] + except Exception: + pass + + res = [re.sub("%.*", "", ip) for ip in res if getIpType(ip) == ip_type and isIp(ip)] + return list(set(res)) + + +def cmp(a, b): + return (a > b) - (a < b) + + +def encodeResponse(func): # Encode returned data from utf8 to bytes + def wrapper(*args, **kwargs): + back = func(*args, **kwargs) + if "__next__" in dir(back): + for part in back: + if type(part) == bytes: + yield part + else: + yield part.encode() + else: + if type(back) == bytes: + yield back + else: + yield back.encode() + + return wrapper diff --git a/start.py b/start.py new file mode 100644 index 00000000..063d7802 --- /dev/null +++ b/start.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + + +# Included modules +import sys + +# ZeroNet Modules +import zeronet + + +def main(): + if "--open_browser" not in sys.argv: + sys.argv = [sys.argv[0]] + ["--open_browser", "default_browser"] + sys.argv[1:] + zeronet.start() + +if __name__ == '__main__': + main() diff --git a/tools/coffee/README.md b/tools/coffee/README.md new file mode 100644 index 00000000..dc589d21 --- /dev/null +++ b/tools/coffee/README.md @@ -0,0 +1,19 @@ +# CoffeeScript compiler for Windows + +A simple command-line utilty for Windows that will compile `*.coffee` files to JavaScript `*.js` files using [CoffeeScript](http://jashkenas.github.com/coffee-script/) and the venerable Windows Script Host, ubiquitous on Windows since the 90s. + +## Usage + +To use it, invoke `coffee.cmd` like so: + + coffee input.coffee output.js + +If an output is not specified, it is written to `stdout`. In neither an input or output are specified then data is assumed to be on `stdin`. For example: + + type input.coffee | coffee > output.js + +Errors are written to `stderr`. + +In the `test` directory there's a version of the standard CoffeeScript tests which can be kicked off using `test.cmd`. The test just attempts to compile the *.coffee files but doesn't execute them. + +To upgrade to the latest CoffeeScript simply replace `coffee-script.js` from the upstream https://github.com/jashkenas/coffee-script/blob/master/extras/coffee-script.js (the tests will likely need updating as well, if you want to run them). diff --git a/tools/coffee/coffee-script.js b/tools/coffee/coffee-script.js new file mode 100644 index 00000000..7fce39a6 --- /dev/null +++ b/tools/coffee/coffee-script.js @@ -0,0 +1,405 @@ +/** + * CoffeeScript Compiler v1.12.6 + * http://coffeescript.org + * + * Copyright 2011, Jeremy Ashkenas + * Released under the MIT License + */ +var $jscomp=$jscomp||{};$jscomp.scope={};$jscomp.checkStringArgs=function(u,xa,va){if(null==u)throw new TypeError("The 'this' value for String.prototype."+va+" must not be null or undefined");if(xa instanceof RegExp)throw new TypeError("First argument to String.prototype."+va+" must not be a regular expression");return u+""}; +$jscomp.defineProperty="function"==typeof Object.defineProperties?Object.defineProperty:function(u,xa,va){if(va.get||va.set)throw new TypeError("ES3 does not support getters and setters.");u!=Array.prototype&&u!=Object.prototype&&(u[xa]=va.value)};$jscomp.getGlobal=function(u){return"undefined"!=typeof window&&window===u?u:"undefined"!=typeof global&&null!=global?global:u};$jscomp.global=$jscomp.getGlobal(this); +$jscomp.polyfill=function(u,xa,va,f){if(xa){va=$jscomp.global;u=u.split(".");for(f=0;fu||1342177279>>=1)va+=va;return f}},"es6-impl","es3");$jscomp.findInternal=function(u,xa,va){u instanceof String&&(u=String(u));for(var f=u.length,qa=0;qa>>=1,a+=a;return g};f.compact=function(a){var g,b;var n=[];var y=0;for(b=a.length;yc)return m.call(this,L,a-1);(w=L[0],0<=y.call(g,w))?c+=1:(l=L[0],0<=y.call(h,l))&&--c;a+=1}return a-1};l.prototype.removeLeadingNewlines=function(){var a,b;var m=this.tokens;var k=a=0;for(b=m.length;ag;f=0<=g?++b:--b){for(;"HERECOMMENT"===this.tag(l+f+c);)c+=2;if(null!=h[f]&&("string"===typeof h[f]&&(h[f]=[h[f]]),k=this.tag(l+f+c),0>y.call(h[f],k)))return-1}return l+f+c-1};l.prototype.looksObjectish=function(a){if(-1y.call(b,w))&&((f=this.tag(a),0>y.call(g,f))||this.tokens[a].generated)&&(n=this.tag(a),0>y.call(R,n)));)(k=this.tag(a),0<=y.call(h,k))&&c.push(this.tag(a)),(l=this.tag(a),0<=y.call(g, +l))&&c.length&&c.pop(),--a;return x=this.tag(a),0<=y.call(b,x)};l.prototype.addImplicitBracesAndParens=function(){var a=[];var l=null;return this.scanTokens(function(c,k,f){var m,w,n,r;var G=c[0];var K=(m=0y.call(h,a):return l[1];case "@"!==this.tag(k-2):return k-2;default:return k-1}}.call(this);"HERECOMMENT"===this.tag(q-2);)q-=2;this.insideForDeclaration="FOR"===u;m=0===q||(r=this.tag(q-1),0<=y.call(R,r))||f[q-1].newLine;if(B()&&(T=B(),r=T[0],v=T[1],("{"===r||"INDENT"===r&&"{"===this.tag(v-1))&&(m||","===this.tag(q-1)||"{"===this.tag(q-1))))return A(1);M(q,!!m);return A(2)}if(0<=y.call(R,G))for(M=a.length-1;0<=M;M+=-1)r=a[M],E(r)&&(r[2].sameLine= +!1);M="OUTDENT"===K||m.newLine;if(0<=y.call(x,G)||0<=y.call(z,G)&&M)for(;O();)if(M=B(),r=M[0],v=M[1],m=M[2],M=m.sameLine,m=m.startsLine,C()&&","!==K)S();else if(T()&&!this.insideForDeclaration&&M&&"TERMINATOR"!==G&&":"!==K)q();else if(!T()||"TERMINATOR"!==G||","===K||m&&this.looksObjectish(k+1))break;else{if("HERECOMMENT"===u)return A(1);q()}if(!(","!==G||this.looksObjectish(k+1)||!T()||this.insideForDeclaration||"TERMINATOR"===u&&this.looksObjectish(k+2)))for(u="OUTDENT"===u?1:0;T();)q(k+u);return A(1)})}; +l.prototype.addLocationDataToGeneratedTokens=function(){return this.scanTokens(function(a,b,g){var c,l;if(a[2]||!a.generated&&!a.explicit)return 1;if("{"===a[0]&&(c=null!=(l=g[b+1])?l[2]:void 0)){var m=c.first_line;c=c.first_column}else(c=null!=(m=g[b-1])?m[2]:void 0)?(m=c.last_line,c=c.last_column):m=c=0;a[2]={first_line:m,first_column:c,last_line:m,last_column:c};return 1})};l.prototype.fixOutdentLocationData=function(){return this.scanTokens(function(a,b,g){if(!("OUTDENT"===a[0]||a.generated&& +"CALL_END"===a[0]||a.generated&&"}"===a[0]))return 1;b=g[b-1][2];a[2]={first_line:b.last_line,first_column:b.last_column,last_line:b.last_line,last_column:b.last_column};return 1})};l.prototype.normalizeLines=function(){var b,g;var l=b=g=null;var k=function(a,b){var c,g,k,f;return";"!==a[1]&&(c=a[0],0<=y.call(O,c))&&!("TERMINATOR"===a[0]&&(g=this.tag(b+1),0<=y.call(H,g)))&&!("ELSE"===a[0]&&"THEN"!==l)&&!!("CATCH"!==(k=a[0])&&"FINALLY"!==k||"-\x3e"!==l&&"\x3d\x3e"!==l)||(f=a[0],0<=y.call(z,f))&&(this.tokens[b- +1].newLine||"OUTDENT"===this.tokens[b-1][0])};var f=function(a,b){return this.tokens.splice(","===this.tag(b-1)?b-1:b,0,g)};return this.scanTokens(function(c,m,h){var w,n,r;c=c[0];if("TERMINATOR"===c){if("ELSE"===this.tag(m+1)&&"OUTDENT"!==this.tag(m-1))return h.splice.apply(h,[m,1].concat(a.call(this.indentation()))),1;if(w=this.tag(m+1),0<=y.call(H,w))return h.splice(m,1),0}if("CATCH"===c)for(w=n=1;2>=n;w=++n)if("OUTDENT"===(r=this.tag(m+w))||"TERMINATOR"===r||"FINALLY"===r)return h.splice.apply(h, +[m+w,0].concat(a.call(this.indentation()))),2+w;0<=y.call(J,c)&&"INDENT"!==this.tag(m+1)&&("ELSE"!==c||"IF"!==this.tag(m+1))&&(l=c,r=this.indentation(h[m]),b=r[0],g=r[1],"THEN"===l&&(b.fromThen=!0),h.splice(m+1,0,b),this.detectEnd(m+2,k,f),"THEN"===c&&h.splice(m,1));return 1})};l.prototype.tagPostfixConditionals=function(){var a=null;var b=function(a,b){a=a[0];b=this.tokens[b-1][0];return"TERMINATOR"===a||"INDENT"===a&&0>y.call(J,b)};var g=function(b,c){if("INDENT"!==b[0]||b.generated&&!b.fromThen)return a[0]= +"POST_"+a[0]};return this.scanTokens(function(c,l){if("IF"!==c[0])return 1;a=c;this.detectEnd(l+1,b,g);return 1})};l.prototype.indentation=function(a){var b=["INDENT",2];var c=["OUTDENT",2];a?(b.generated=c.generated=!0,b.origin=c.origin=a):b.explicit=c.explicit=!0;return[b,c]};l.prototype.generate=b;l.prototype.tag=function(a){var b;return null!=(b=this.tokens[a])?b[0]:void 0};return l}();var ya=[["(",")"],["[","]"],["{","}"],["INDENT","OUTDENT"],["CALL_START","CALL_END"],["PARAM_START","PARAM_END"], +["INDEX_START","INDEX_END"],["STRING_START","STRING_END"],["REGEX_START","REGEX_END"]];f.INVERSES=u={};var g=[];var h=[];var r=0;for(q=ya.length;rthis.indent){if(c||"RETURN"===this.tag())return this.indebt=b-this.indent,this.suppressNewlines(),a.length;if(!this.tokens.length)return this.baseIndent= +this.indent=b,a.length;c=b-this.indent+this.outdebt;this.token("INDENT",c,a.length-b,b);this.indents.push(c);this.ends.push({tag:"OUTDENT"});this.outdebt=this.indebt=0;this.indent=b}else bl&&(m=this.token("+","+"),m[2]={first_line:w[2].first_line,first_column:w[2].first_column,last_line:w[2].first_line,last_column:w[2].first_column});(f=this.tokens).push.apply(f,r)}if(k)return a=a[a.length-1],k.origin=["STRING",null,{first_line:k[2].first_line,first_column:k[2].first_column,last_line:a[2].last_line,last_column:a[2].last_column}],k=this.token("STRING_END",")"),k[2]={first_line:a[2].last_line,first_column:a[2].last_column, +last_line:a[2].last_line,last_column:a[2].last_column}};a.prototype.pair=function(a){var b=this.ends;b=b[b.length-1];return a!==(b=null!=b?b.tag:void 0)?("OUTDENT"!==b&&this.error("unmatched "+a),b=this.indents,b=b[b.length-1],this.outdentToken(b,!0),this.pair(a)):this.ends.pop()};a.prototype.getLineAndColumnFromChunk=function(a){if(0===a)return[this.chunkLine,this.chunkColumn];var b=a>=this.chunk.length?this.chunk:this.chunk.slice(0,+(a-1)+1||9E9);a=g(b,"\n");var c=this.chunkColumn;0a)return b(a);var c=Math.floor((a-65536)/1024)+55296;a=(a-65536)%1024+56320;return""+b(c)+b(a)};a.prototype.replaceUnicodeCodePointEscapes= +function(a,b){return a.replace(sa,function(a){return function(c,g,k,h){if(g)return g;c=parseInt(k,16);1114111q.call(y.call(I).concat(y.call(F)),a):return"keyword '"+b+"' can't be assigned";case 0>q.call(O, +a):return"'"+b+"' can't be assigned";case 0>q.call(J,a):return"reserved word '"+b+"' can't be assigned";default:return!1}};f.isUnassignable=B;var H=function(a){var b;return"IDENTIFIER"===a[0]?("from"===a[1]&&(a[1][0]="IDENTIFIER",!0),!0):"FOR"===a[0]?!1:"{"===(b=a[1])||"["===b||","===b||":"===b?!1:!0};var I="true false null this new delete typeof in instanceof return throw break continue debugger yield if else switch for while do try catch finally class extends super import export default".split(" "); +var F="undefined Infinity NaN then unless until loop of by when".split(" ");var Q={and:"\x26\x26",or:"||",is:"\x3d\x3d",isnt:"!\x3d",not:"!",yes:"true",no:"false",on:"true",off:"false"};var x=function(){var a=[];for(qa in Q)a.push(qa);return a}();F=F.concat(x);var J="case function var void with const let enum native implements interface package private protected public static".split(" ");var O=["arguments","eval"];f.JS_FORBIDDEN=I.concat(J).concat(O);var R=65279;var z=/^(?!\d)((?:(?!\s)[$\w\x7f-\uffff])+)([^\n\S]*:(?!:))?/; +var l=/^0b[01]+|^0o[0-7]+|^0x[\da-f]+|^\d*\.?\d+(?:e[+-]?\d+)?/i;var c=/^(?:[-=]>|[-+*\/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>*\/%])\2=?|\?(\.|::)|\.{2,3})/;var w=/^[^\n\S]+/;var m=/^###([^#][\s\S]*?)(?:###[^\n\S]*|###$)|^(?:\s*#(?!##[^#]).*)+/;var k=/^[-=]>/;var K=/^(?:\n[^\n\S]*)+/;var P=/^`(?!``)((?:[^`\\]|\\[\s\S])*)`/;var L=/^```((?:[^`\\]|\\[\s\S]|`(?!``))*)```/;var V=/^(?:'''|"""|'|")/;var X=/^(?:[^\\']|\\[\s\S])*/;var G=/^(?:[^\\"#]|\\[\s\S]|\#(?!\{))*/;var aa=/^(?:[^\\']|\\[\s\S]|'(?!''))*/; +var U=/^(?:[^\\"#]|\\[\s\S]|"(?!"")|\#(?!\{))*/;var W=/((?:\\\\)+)|\\[^\S\n]*\n\s*/g;var D=/\s*\n\s*/g;var A=/\n+([^\n\S]*)(?=\S)/g;var fc=/^\/(?!\/)((?:[^[\/\n\\]|\\[^\n]|\[(?:\\[^\n]|[^\]\n\\])*\])*)(\/)?/;var E=/^\w*/;var ba=/^(?!.*(.).*\1)[imguy]*$/;var ca=/^(?:[^\\\/#]|\\[\s\S]|\/(?!\/\/)|\#(?!\{))*/;var C=/((?:\\\\)+)|\\(\s)|\s+(?:#.*)?/g;var T=/^(\/|\/{3}\s*)(\*)/;var v=/^\/=?\s/;var Y=/\*\//;var S=/^\s*(?:,|\??\.(?![.\d])|::)/;var M=/((?:^|[^\\])(?:\\\\)*)\\(?:(0[0-7]|[1-7])|(x(?![\da-fA-F]{2}).{0,2})|(u\{(?![\da-fA-F]{1,}\})[^}]*\}?)|(u(?!\{|[\da-fA-F]{4}).{0,4}))/; +var va=/((?:^|[^\\])(?:\\\\)*)\\(?:(0[0-7])|(x(?![\da-fA-F]{2}).{0,2})|(u\{(?![\da-fA-F]{1,}\})[^}]*\}?)|(u(?!\{|[\da-fA-F]{4}).{0,4}))/;var sa=/(\\\\)|\\u\{([\da-fA-F]+)\}/g;var za=/^[^\n\S]*\n/;var ma=/\n[^\n\S]*$/;var Z=/\s+$/;var fa="-\x3d +\x3d /\x3d *\x3d %\x3d ||\x3d \x26\x26\x3d ?\x3d \x3c\x3c\x3d \x3e\x3e\x3d \x3e\x3e\x3e\x3d \x26\x3d ^\x3d |\x3d **\x3d //\x3d %%\x3d".split(" ");var ia=["NEW","TYPEOF","DELETE","DO"];var ga=["!","~"];var ja=["\x3c\x3c","\x3e\x3e","\x3e\x3e\x3e"];var la="\x3d\x3d !\x3d \x3c \x3e \x3c\x3d \x3e\x3d".split(" "); +var oa=["*","/","%","//","%%"];var pa=["IN","OF","INSTANCEOF"];var ha="IDENTIFIER PROPERTY ) ] ? @ THIS SUPER".split(" ");var ka=ha.concat("NUMBER INFINITY NAN STRING STRING_END REGEX REGEX_END BOOL NULL UNDEFINED } ::".split(" "));var na=ka.concat(["++","--"]);var ra=["INDENT","OUTDENT","TERMINATOR"];var da=[")","}","]"]}).call(this);return f}();u["./parser"]=function(){var f={},qa={exports:f},q=function(){function f(){this.yy={}}var a=function(a,p,t,d){t=t||{};for(d=a.length;d--;t[a[d]]=p);return t}, +b=[1,22],u=[1,25],g=[1,83],h=[1,79],r=[1,84],n=[1,85],B=[1,81],H=[1,82],I=[1,56],F=[1,58],Q=[1,59],x=[1,60],J=[1,61],O=[1,62],R=[1,49],z=[1,50],l=[1,32],c=[1,68],w=[1,69],m=[1,78],k=[1,47],K=[1,51],P=[1,52],L=[1,67],V=[1,65],X=[1,66],G=[1,64],aa=[1,42],U=[1,48],W=[1,63],D=[1,73],A=[1,74],q=[1,75],E=[1,76],ba=[1,46],ca=[1,72],C=[1,34],T=[1,35],v=[1,36],Y=[1,37],S=[1,38],M=[1,39],qa=[1,86],sa=[1,6,32,42,131],za=[1,101],ma=[1,89],Z=[1,88],fa=[1,87],ia=[1,90],ga=[1,91],ja=[1,92],la=[1,93],oa=[1,94],pa= +[1,95],ha=[1,96],ka=[1,97],na=[1,98],ra=[1,99],da=[1,100],va=[1,104],N=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],xa=[2,166],ta=[1,110],Na=[1,111],Fa=[1,112],Ga=[1,113],Ca=[1,115],Pa=[1,116],Ia=[1,109],Ea=[1,6,32,42,131,133,135,139,156],Va=[2,27],ea=[1,123],Ya=[1,121],Ba=[1,6,31,32,40,41,42,65,70,73,82,83,84,85,87,89,90,94,113,114,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172, +173,174],Ha=[2,94],t=[1,6,31,32,42,46,65,70,73,82,83,84,85,87,89,90,94,113,114,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],p=[2,73],d=[1,128],wa=[1,133],e=[1,134],Da=[1,136],Ta=[1,6,31,32,40,41,42,55,65,70,73,82,83,84,85,87,89,90,94,113,114,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],ua=[2,91],Eb=[1,6,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168, +169,170,171,172,173,174],Za=[2,63],Fb=[1,166],$a=[1,178],Ua=[1,180],Gb=[1,175],Oa=[1,182],sb=[1,184],La=[1,6,31,32,40,41,42,55,65,70,73,82,83,84,85,87,89,90,94,96,113,114,115,120,122,131,133,134,135,139,140,156,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175],Hb=[2,110],Ib=[1,6,31,32,40,41,42,58,65,70,73,82,83,84,85,87,89,90,94,113,114,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],Jb=[1,6,31,32,40,41,42,46,58,65,70,73,82,83,84, +85,87,89,90,94,113,114,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],Kb=[40,41,114],Lb=[1,241],tb=[1,240],Ma=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156],Ja=[2,71],Mb=[1,250],Sa=[6,31,32,65,70],fb=[6,31,32,55,65,70,73],ab=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,159,160,164,166,167,168,169,170,171,172,173,174],Nb=[40,41,82,83,84,85,87,90,113,114],gb=[1,269],bb=[2,62],hb=[1,279],Wa=[1,281],ub=[1, +286],cb=[1,288],Ob=[2,187],vb=[1,6,31,32,40,41,42,55,65,70,73,82,83,84,85,87,89,90,94,113,114,115,120,122,131,133,134,135,139,140,146,147,148,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],ib=[1,297],Qa=[6,31,32,70,115,120],Pb=[1,6,31,32,40,41,42,55,58,65,70,73,82,83,84,85,87,89,90,94,96,113,114,115,120,122,131,133,134,135,139,140,146,147,148,156,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175],Qb=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,140,156],Xa=[1,6,31,32, +42,65,70,73,89,94,115,120,122,131,134,140,156],jb=[146,147,148],kb=[70,146,147,148],lb=[6,31,94],Rb=[1,311],Aa=[6,31,32,70,94],Sb=[6,31,32,58,70,94],wb=[6,31,32,55,58,70,94],Tb=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,159,160,166,167,168,169,170,171,172,173,174],Ub=[12,28,34,38,40,41,44,45,48,49,50,51,52,53,61,62,63,67,68,89,92,95,97,105,112,117,118,119,125,129,130,133,135,137,139,149,155,157,158,159,160,161,162],Vb=[2,176],Ra=[6,31,32],db=[2,72],Wb=[1,323],Xb=[1,324], +Yb=[1,6,31,32,42,65,70,73,89,94,115,120,122,127,128,131,133,134,135,139,140,151,153,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],mb=[32,151,153],Zb=[1,6,32,42,65,70,73,89,94,115,120,122,131,134,140,156],nb=[1,350],xb=[1,356],yb=[1,6,32,42,131,156],eb=[2,86],ob=[1,367],pb=[1,368],$b=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,151,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],zb=[1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,135,139,140,156],ac= +[1,381],bc=[1,382],Ab=[6,31,32,94],cc=[6,31,32,70],Bb=[1,6,31,32,42,65,70,73,89,94,115,120,122,127,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],dc=[31,70],qb=[1,408],rb=[1,409],Cb=[1,415],Db=[1,416],ec={trace:function(){},yy:{},symbols_:{error:2,Root:3,Body:4,Line:5,TERMINATOR:6,Expression:7,Statement:8,YieldReturn:9,Return:10,Comment:11,STATEMENT:12,Import:13,Export:14,Value:15,Invocation:16,Code:17,Operation:18,Assign:19,If:20,Try:21,While:22,For:23,Switch:24, +Class:25,Throw:26,Yield:27,YIELD:28,FROM:29,Block:30,INDENT:31,OUTDENT:32,Identifier:33,IDENTIFIER:34,Property:35,PROPERTY:36,AlphaNumeric:37,NUMBER:38,String:39,STRING:40,STRING_START:41,STRING_END:42,Regex:43,REGEX:44,REGEX_START:45,REGEX_END:46,Literal:47,JS:48,UNDEFINED:49,NULL:50,BOOL:51,INFINITY:52,NAN:53,Assignable:54,"\x3d":55,AssignObj:56,ObjAssignable:57,":":58,SimpleObjAssignable:59,ThisProperty:60,RETURN:61,HERECOMMENT:62,PARAM_START:63,ParamList:64,PARAM_END:65,FuncGlyph:66,"-\x3e":67, +"\x3d\x3e":68,OptComma:69,",":70,Param:71,ParamVar:72,"...":73,Array:74,Object:75,Splat:76,SimpleAssignable:77,Accessor:78,Parenthetical:79,Range:80,This:81,".":82,"?.":83,"::":84,"?::":85,Index:86,INDEX_START:87,IndexValue:88,INDEX_END:89,INDEX_SOAK:90,Slice:91,"{":92,AssignList:93,"}":94,CLASS:95,EXTENDS:96,IMPORT:97,ImportDefaultSpecifier:98,ImportNamespaceSpecifier:99,ImportSpecifierList:100,ImportSpecifier:101,AS:102,DEFAULT:103,IMPORT_ALL:104,EXPORT:105,ExportSpecifierList:106,EXPORT_ALL:107, +ExportSpecifier:108,OptFuncExist:109,Arguments:110,Super:111,SUPER:112,FUNC_EXIST:113,CALL_START:114,CALL_END:115,ArgList:116,THIS:117,"@":118,"[":119,"]":120,RangeDots:121,"..":122,Arg:123,SimpleArgs:124,TRY:125,Catch:126,FINALLY:127,CATCH:128,THROW:129,"(":130,")":131,WhileSource:132,WHILE:133,WHEN:134,UNTIL:135,Loop:136,LOOP:137,ForBody:138,FOR:139,BY:140,ForStart:141,ForSource:142,ForVariables:143,OWN:144,ForValue:145,FORIN:146,FOROF:147,FORFROM:148,SWITCH:149,Whens:150,ELSE:151,When:152,LEADING_WHEN:153, +IfBlock:154,IF:155,POST_IF:156,UNARY:157,UNARY_MATH:158,"-":159,"+":160,"--":161,"++":162,"?":163,MATH:164,"**":165,SHIFT:166,COMPARE:167,"\x26":168,"^":169,"|":170,"\x26\x26":171,"||":172,"BIN?":173,RELATION:174,COMPOUND_ASSIGN:175,$accept:0,$end:1},terminals_:{2:"error",6:"TERMINATOR",12:"STATEMENT",28:"YIELD",29:"FROM",31:"INDENT",32:"OUTDENT",34:"IDENTIFIER",36:"PROPERTY",38:"NUMBER",40:"STRING",41:"STRING_START",42:"STRING_END",44:"REGEX",45:"REGEX_START",46:"REGEX_END",48:"JS",49:"UNDEFINED", +50:"NULL",51:"BOOL",52:"INFINITY",53:"NAN",55:"\x3d",58:":",61:"RETURN",62:"HERECOMMENT",63:"PARAM_START",65:"PARAM_END",67:"-\x3e",68:"\x3d\x3e",70:",",73:"...",82:".",83:"?.",84:"::",85:"?::",87:"INDEX_START",89:"INDEX_END",90:"INDEX_SOAK",92:"{",94:"}",95:"CLASS",96:"EXTENDS",97:"IMPORT",102:"AS",103:"DEFAULT",104:"IMPORT_ALL",105:"EXPORT",107:"EXPORT_ALL",112:"SUPER",113:"FUNC_EXIST",114:"CALL_START",115:"CALL_END",117:"THIS",118:"@",119:"[",120:"]",122:"..",125:"TRY",127:"FINALLY",128:"CATCH", +129:"THROW",130:"(",131:")",133:"WHILE",134:"WHEN",135:"UNTIL",137:"LOOP",139:"FOR",140:"BY",144:"OWN",146:"FORIN",147:"FOROF",148:"FORFROM",149:"SWITCH",151:"ELSE",153:"LEADING_WHEN",155:"IF",156:"POST_IF",157:"UNARY",158:"UNARY_MATH",159:"-",160:"+",161:"--",162:"++",163:"?",164:"MATH",165:"**",166:"SHIFT",167:"COMPARE",168:"\x26",169:"^",170:"|",171:"\x26\x26",172:"||",173:"BIN?",174:"RELATION",175:"COMPOUND_ASSIGN"},productions_:[0,[3,0],[3,1],[4,1],[4,3],[4,2],[5,1],[5,1],[5,1],[8,1],[8,1],[8, +1],[8,1],[8,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[7,1],[27,1],[27,2],[27,3],[30,2],[30,3],[33,1],[35,1],[37,1],[37,1],[39,1],[39,3],[43,1],[43,3],[47,1],[47,1],[47,1],[47,1],[47,1],[47,1],[47,1],[47,1],[19,3],[19,4],[19,5],[56,1],[56,3],[56,5],[56,3],[56,5],[56,1],[59,1],[59,1],[59,1],[57,1],[57,1],[10,2],[10,1],[9,3],[9,2],[11,1],[17,5],[17,2],[66,1],[66,1],[69,0],[69,1],[64,0],[64,1],[64,3],[64,4],[64,6],[71,1],[71,2],[71,3],[71,1],[72,1],[72,1],[72,1],[72, +1],[76,2],[77,1],[77,2],[77,2],[77,1],[54,1],[54,1],[54,1],[15,1],[15,1],[15,1],[15,1],[15,1],[78,2],[78,2],[78,2],[78,2],[78,1],[78,1],[86,3],[86,2],[88,1],[88,1],[75,4],[93,0],[93,1],[93,3],[93,4],[93,6],[25,1],[25,2],[25,3],[25,4],[25,2],[25,3],[25,4],[25,5],[13,2],[13,4],[13,4],[13,5],[13,7],[13,6],[13,9],[100,1],[100,3],[100,4],[100,4],[100,6],[101,1],[101,3],[101,1],[101,3],[98,1],[99,3],[14,3],[14,5],[14,2],[14,4],[14,5],[14,6],[14,3],[14,4],[14,7],[106,1],[106,3],[106,4],[106,4],[106,6],[108, +1],[108,3],[108,3],[108,1],[108,3],[16,3],[16,3],[16,3],[16,1],[111,1],[111,2],[109,0],[109,1],[110,2],[110,4],[81,1],[81,1],[60,2],[74,2],[74,4],[121,1],[121,1],[80,5],[91,3],[91,2],[91,2],[91,1],[116,1],[116,3],[116,4],[116,4],[116,6],[123,1],[123,1],[123,1],[124,1],[124,3],[21,2],[21,3],[21,4],[21,5],[126,3],[126,3],[126,2],[26,2],[79,3],[79,5],[132,2],[132,4],[132,2],[132,4],[22,2],[22,2],[22,2],[22,1],[136,2],[136,2],[23,2],[23,2],[23,2],[138,2],[138,4],[138,2],[141,2],[141,3],[145,1],[145,1], +[145,1],[145,1],[143,1],[143,3],[142,2],[142,2],[142,4],[142,4],[142,4],[142,6],[142,6],[142,2],[142,4],[24,5],[24,7],[24,4],[24,6],[150,1],[150,2],[152,3],[152,4],[154,3],[154,5],[20,1],[20,3],[20,3],[20,3],[18,2],[18,2],[18,2],[18,2],[18,2],[18,2],[18,2],[18,2],[18,2],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,3],[18,5],[18,4],[18,3]],performAction:function(a,p,t,d,wa,b,e){a=b.length-1;switch(wa){case 1:return this.$=d.addLocationDataFn(e[a],e[a])(new d.Block); +case 2:return this.$=b[a];case 3:this.$=d.addLocationDataFn(e[a],e[a])(d.Block.wrap([b[a]]));break;case 4:this.$=d.addLocationDataFn(e[a-2],e[a])(b[a-2].push(b[a]));break;case 5:this.$=b[a-1];break;case 6:case 7:case 8:case 9:case 10:case 12:case 13:case 14:case 15:case 16:case 17:case 18:case 19:case 20:case 21:case 22:case 23:case 24:case 25:case 26:case 35:case 40:case 42:case 56:case 57:case 58:case 59:case 60:case 61:case 71:case 72:case 82:case 83:case 84:case 85:case 90:case 91:case 94:case 98:case 104:case 163:case 187:case 188:case 190:case 220:case 221:case 239:case 245:this.$= +b[a];break;case 11:this.$=d.addLocationDataFn(e[a],e[a])(new d.StatementLiteral(b[a]));break;case 27:this.$=d.addLocationDataFn(e[a],e[a])(new d.Op(b[a],new d.Value(new d.Literal(""))));break;case 28:case 249:case 250:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op(b[a-1],b[a]));break;case 29:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Op(b[a-2].concat(b[a-1]),b[a]));break;case 30:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Block);break;case 31:case 105:this.$=d.addLocationDataFn(e[a-2],e[a])(b[a- +1]);break;case 32:this.$=d.addLocationDataFn(e[a],e[a])(new d.IdentifierLiteral(b[a]));break;case 33:this.$=d.addLocationDataFn(e[a],e[a])(new d.PropertyName(b[a]));break;case 34:this.$=d.addLocationDataFn(e[a],e[a])(new d.NumberLiteral(b[a]));break;case 36:this.$=d.addLocationDataFn(e[a],e[a])(new d.StringLiteral(b[a]));break;case 37:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.StringWithInterpolations(b[a-1]));break;case 38:this.$=d.addLocationDataFn(e[a],e[a])(new d.RegexLiteral(b[a]));break; +case 39:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.RegexWithInterpolations(b[a-1].args));break;case 41:this.$=d.addLocationDataFn(e[a],e[a])(new d.PassthroughLiteral(b[a]));break;case 43:this.$=d.addLocationDataFn(e[a],e[a])(new d.UndefinedLiteral);break;case 44:this.$=d.addLocationDataFn(e[a],e[a])(new d.NullLiteral);break;case 45:this.$=d.addLocationDataFn(e[a],e[a])(new d.BooleanLiteral(b[a]));break;case 46:this.$=d.addLocationDataFn(e[a],e[a])(new d.InfinityLiteral(b[a]));break;case 47:this.$= +d.addLocationDataFn(e[a],e[a])(new d.NaNLiteral);break;case 48:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Assign(b[a-2],b[a]));break;case 49:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Assign(b[a-3],b[a]));break;case 50:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Assign(b[a-4],b[a-1]));break;case 51:case 87:case 92:case 93:case 95:case 96:case 97:case 222:case 223:this.$=d.addLocationDataFn(e[a],e[a])(new d.Value(b[a]));break;case 52:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Assign(d.addLocationDataFn(e[a- +2])(new d.Value(b[a-2])),b[a],"object",{operatorToken:d.addLocationDataFn(e[a-1])(new d.Literal(b[a-1]))}));break;case 53:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Assign(d.addLocationDataFn(e[a-4])(new d.Value(b[a-4])),b[a-1],"object",{operatorToken:d.addLocationDataFn(e[a-3])(new d.Literal(b[a-3]))}));break;case 54:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Assign(d.addLocationDataFn(e[a-2])(new d.Value(b[a-2])),b[a],null,{operatorToken:d.addLocationDataFn(e[a-1])(new d.Literal(b[a-1]))})); +break;case 55:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Assign(d.addLocationDataFn(e[a-4])(new d.Value(b[a-4])),b[a-1],null,{operatorToken:d.addLocationDataFn(e[a-3])(new d.Literal(b[a-3]))}));break;case 62:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Return(b[a]));break;case 63:this.$=d.addLocationDataFn(e[a],e[a])(new d.Return);break;case 64:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.YieldReturn(b[a]));break;case 65:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.YieldReturn);break;case 66:this.$= +d.addLocationDataFn(e[a],e[a])(new d.Comment(b[a]));break;case 67:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Code(b[a-3],b[a],b[a-1]));break;case 68:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Code([],b[a],b[a-1]));break;case 69:this.$=d.addLocationDataFn(e[a],e[a])("func");break;case 70:this.$=d.addLocationDataFn(e[a],e[a])("boundfunc");break;case 73:case 110:this.$=d.addLocationDataFn(e[a],e[a])([]);break;case 74:case 111:case 130:case 150:case 182:case 224:this.$=d.addLocationDataFn(e[a], +e[a])([b[a]]);break;case 75:case 112:case 131:case 151:case 183:this.$=d.addLocationDataFn(e[a-2],e[a])(b[a-2].concat(b[a]));break;case 76:case 113:case 132:case 152:case 184:this.$=d.addLocationDataFn(e[a-3],e[a])(b[a-3].concat(b[a]));break;case 77:case 114:case 134:case 154:case 186:this.$=d.addLocationDataFn(e[a-5],e[a])(b[a-5].concat(b[a-2]));break;case 78:this.$=d.addLocationDataFn(e[a],e[a])(new d.Param(b[a]));break;case 79:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Param(b[a-1],null,!0)); +break;case 80:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Param(b[a-2],b[a]));break;case 81:case 189:this.$=d.addLocationDataFn(e[a],e[a])(new d.Expansion);break;case 86:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Splat(b[a-1]));break;case 88:this.$=d.addLocationDataFn(e[a-1],e[a])(b[a-1].add(b[a]));break;case 89:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Value(b[a-1],[].concat(b[a])));break;case 99:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Access(b[a]));break;case 100:this.$=d.addLocationDataFn(e[a- +1],e[a])(new d.Access(b[a],"soak"));break;case 101:this.$=d.addLocationDataFn(e[a-1],e[a])([d.addLocationDataFn(e[a-1])(new d.Access(new d.PropertyName("prototype"))),d.addLocationDataFn(e[a])(new d.Access(b[a]))]);break;case 102:this.$=d.addLocationDataFn(e[a-1],e[a])([d.addLocationDataFn(e[a-1])(new d.Access(new d.PropertyName("prototype"),"soak")),d.addLocationDataFn(e[a])(new d.Access(b[a]))]);break;case 103:this.$=d.addLocationDataFn(e[a],e[a])(new d.Access(new d.PropertyName("prototype"))); +break;case 106:this.$=d.addLocationDataFn(e[a-1],e[a])(d.extend(b[a],{soak:!0}));break;case 107:this.$=d.addLocationDataFn(e[a],e[a])(new d.Index(b[a]));break;case 108:this.$=d.addLocationDataFn(e[a],e[a])(new d.Slice(b[a]));break;case 109:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Obj(b[a-2],b[a-3].generated));break;case 115:this.$=d.addLocationDataFn(e[a],e[a])(new d.Class);break;case 116:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Class(null,null,b[a]));break;case 117:this.$=d.addLocationDataFn(e[a- +2],e[a])(new d.Class(null,b[a]));break;case 118:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Class(null,b[a-1],b[a]));break;case 119:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Class(b[a]));break;case 120:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Class(b[a-1],null,b[a]));break;case 121:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Class(b[a-2],b[a]));break;case 122:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Class(b[a-3],b[a-1],b[a]));break;case 123:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.ImportDeclaration(null, +b[a]));break;case 124:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.ImportDeclaration(new d.ImportClause(b[a-2],null),b[a]));break;case 125:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.ImportDeclaration(new d.ImportClause(null,b[a-2]),b[a]));break;case 126:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.ImportDeclaration(new d.ImportClause(null,new d.ImportSpecifierList([])),b[a]));break;case 127:this.$=d.addLocationDataFn(e[a-6],e[a])(new d.ImportDeclaration(new d.ImportClause(null,new d.ImportSpecifierList(b[a- +4])),b[a]));break;case 128:this.$=d.addLocationDataFn(e[a-5],e[a])(new d.ImportDeclaration(new d.ImportClause(b[a-4],b[a-2]),b[a]));break;case 129:this.$=d.addLocationDataFn(e[a-8],e[a])(new d.ImportDeclaration(new d.ImportClause(b[a-7],new d.ImportSpecifierList(b[a-4])),b[a]));break;case 133:case 153:case 169:case 185:this.$=d.addLocationDataFn(e[a-3],e[a])(b[a-2]);break;case 135:this.$=d.addLocationDataFn(e[a],e[a])(new d.ImportSpecifier(b[a]));break;case 136:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ImportSpecifier(b[a- +2],b[a]));break;case 137:this.$=d.addLocationDataFn(e[a],e[a])(new d.ImportSpecifier(new d.Literal(b[a])));break;case 138:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ImportSpecifier(new d.Literal(b[a-2]),b[a]));break;case 139:this.$=d.addLocationDataFn(e[a],e[a])(new d.ImportDefaultSpecifier(b[a]));break;case 140:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ImportNamespaceSpecifier(new d.Literal(b[a-2]),b[a]));break;case 141:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ExportNamedDeclaration(new d.ExportSpecifierList([]))); +break;case 142:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.ExportNamedDeclaration(new d.ExportSpecifierList(b[a-2])));break;case 143:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.ExportNamedDeclaration(b[a]));break;case 144:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.ExportNamedDeclaration(new d.Assign(b[a-2],b[a],null,{moduleDeclaration:"export"})));break;case 145:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.ExportNamedDeclaration(new d.Assign(b[a-3],b[a],null,{moduleDeclaration:"export"}))); +break;case 146:this.$=d.addLocationDataFn(e[a-5],e[a])(new d.ExportNamedDeclaration(new d.Assign(b[a-4],b[a-1],null,{moduleDeclaration:"export"})));break;case 147:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ExportDefaultDeclaration(b[a]));break;case 148:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.ExportAllDeclaration(new d.Literal(b[a-2]),b[a]));break;case 149:this.$=d.addLocationDataFn(e[a-6],e[a])(new d.ExportNamedDeclaration(new d.ExportSpecifierList(b[a-4]),b[a]));break;case 155:this.$=d.addLocationDataFn(e[a], +e[a])(new d.ExportSpecifier(b[a]));break;case 156:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ExportSpecifier(b[a-2],b[a]));break;case 157:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ExportSpecifier(b[a-2],new d.Literal(b[a])));break;case 158:this.$=d.addLocationDataFn(e[a],e[a])(new d.ExportSpecifier(new d.Literal(b[a])));break;case 159:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.ExportSpecifier(new d.Literal(b[a-2]),b[a]));break;case 160:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.TaggedTemplateCall(b[a- +2],b[a],b[a-1]));break;case 161:case 162:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Call(b[a-2],b[a],b[a-1]));break;case 164:this.$=d.addLocationDataFn(e[a],e[a])(new d.SuperCall);break;case 165:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.SuperCall(b[a]));break;case 166:this.$=d.addLocationDataFn(e[a],e[a])(!1);break;case 167:this.$=d.addLocationDataFn(e[a],e[a])(!0);break;case 168:this.$=d.addLocationDataFn(e[a-1],e[a])([]);break;case 170:case 171:this.$=d.addLocationDataFn(e[a],e[a])(new d.Value(new d.ThisLiteral)); +break;case 172:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Value(d.addLocationDataFn(e[a-1])(new d.ThisLiteral),[d.addLocationDataFn(e[a])(new d.Access(b[a]))],"this"));break;case 173:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Arr([]));break;case 174:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Arr(b[a-2]));break;case 175:this.$=d.addLocationDataFn(e[a],e[a])("inclusive");break;case 176:this.$=d.addLocationDataFn(e[a],e[a])("exclusive");break;case 177:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Range(b[a- +3],b[a-1],b[a-2]));break;case 178:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Range(b[a-2],b[a],b[a-1]));break;case 179:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Range(b[a-1],null,b[a]));break;case 180:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Range(null,b[a],b[a-1]));break;case 181:this.$=d.addLocationDataFn(e[a],e[a])(new d.Range(null,null,b[a]));break;case 191:this.$=d.addLocationDataFn(e[a-2],e[a])([].concat(b[a-2],b[a]));break;case 192:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Try(b[a])); +break;case 193:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Try(b[a-1],b[a][0],b[a][1]));break;case 194:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Try(b[a-2],null,null,b[a]));break;case 195:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Try(b[a-3],b[a-2][0],b[a-2][1],b[a]));break;case 196:this.$=d.addLocationDataFn(e[a-2],e[a])([b[a-1],b[a]]);break;case 197:this.$=d.addLocationDataFn(e[a-2],e[a])([d.addLocationDataFn(e[a-1])(new d.Value(b[a-1])),b[a]]);break;case 198:this.$=d.addLocationDataFn(e[a- +1],e[a])([null,b[a]]);break;case 199:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Throw(b[a]));break;case 200:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Parens(b[a-1]));break;case 201:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Parens(b[a-2]));break;case 202:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.While(b[a]));break;case 203:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.While(b[a-2],{guard:b[a]}));break;case 204:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.While(b[a],{invert:!0}));break; +case 205:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.While(b[a-2],{invert:!0,guard:b[a]}));break;case 206:this.$=d.addLocationDataFn(e[a-1],e[a])(b[a-1].addBody(b[a]));break;case 207:case 208:this.$=d.addLocationDataFn(e[a-1],e[a])(b[a].addBody(d.addLocationDataFn(e[a-1])(d.Block.wrap([b[a-1]]))));break;case 209:this.$=d.addLocationDataFn(e[a],e[a])(b[a]);break;case 210:this.$=d.addLocationDataFn(e[a-1],e[a])((new d.While(d.addLocationDataFn(e[a-1])(new d.BooleanLiteral("true")))).addBody(b[a])); +break;case 211:this.$=d.addLocationDataFn(e[a-1],e[a])((new d.While(d.addLocationDataFn(e[a-1])(new d.BooleanLiteral("true")))).addBody(d.addLocationDataFn(e[a])(d.Block.wrap([b[a]]))));break;case 212:case 213:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.For(b[a-1],b[a]));break;case 214:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.For(b[a],b[a-1]));break;case 215:this.$=d.addLocationDataFn(e[a-1],e[a])({source:d.addLocationDataFn(e[a])(new d.Value(b[a]))});break;case 216:this.$=d.addLocationDataFn(e[a- +3],e[a])({source:d.addLocationDataFn(e[a-2])(new d.Value(b[a-2])),step:b[a]});break;case 217:d=d.addLocationDataFn(e[a-1],e[a]);b[a].own=b[a-1].own;b[a].ownTag=b[a-1].ownTag;b[a].name=b[a-1][0];b[a].index=b[a-1][1];this.$=d(b[a]);break;case 218:this.$=d.addLocationDataFn(e[a-1],e[a])(b[a]);break;case 219:wa=d.addLocationDataFn(e[a-2],e[a]);b[a].own=!0;b[a].ownTag=d.addLocationDataFn(e[a-1])(new d.Literal(b[a-1]));this.$=wa(b[a]);break;case 225:this.$=d.addLocationDataFn(e[a-2],e[a])([b[a-2],b[a]]); +break;case 226:this.$=d.addLocationDataFn(e[a-1],e[a])({source:b[a]});break;case 227:this.$=d.addLocationDataFn(e[a-1],e[a])({source:b[a],object:!0});break;case 228:this.$=d.addLocationDataFn(e[a-3],e[a])({source:b[a-2],guard:b[a]});break;case 229:this.$=d.addLocationDataFn(e[a-3],e[a])({source:b[a-2],guard:b[a],object:!0});break;case 230:this.$=d.addLocationDataFn(e[a-3],e[a])({source:b[a-2],step:b[a]});break;case 231:this.$=d.addLocationDataFn(e[a-5],e[a])({source:b[a-4],guard:b[a-2],step:b[a]}); +break;case 232:this.$=d.addLocationDataFn(e[a-5],e[a])({source:b[a-4],step:b[a-2],guard:b[a]});break;case 233:this.$=d.addLocationDataFn(e[a-1],e[a])({source:b[a],from:!0});break;case 234:this.$=d.addLocationDataFn(e[a-3],e[a])({source:b[a-2],guard:b[a],from:!0});break;case 235:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Switch(b[a-3],b[a-1]));break;case 236:this.$=d.addLocationDataFn(e[a-6],e[a])(new d.Switch(b[a-5],b[a-3],b[a-1]));break;case 237:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Switch(null, +b[a-1]));break;case 238:this.$=d.addLocationDataFn(e[a-5],e[a])(new d.Switch(null,b[a-3],b[a-1]));break;case 240:this.$=d.addLocationDataFn(e[a-1],e[a])(b[a-1].concat(b[a]));break;case 241:this.$=d.addLocationDataFn(e[a-2],e[a])([[b[a-1],b[a]]]);break;case 242:this.$=d.addLocationDataFn(e[a-3],e[a])([[b[a-2],b[a-1]]]);break;case 243:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.If(b[a-1],b[a],{type:b[a-2]}));break;case 244:this.$=d.addLocationDataFn(e[a-4],e[a])(b[a-4].addElse(d.addLocationDataFn(e[a- +2],e[a])(new d.If(b[a-1],b[a],{type:b[a-2]}))));break;case 246:this.$=d.addLocationDataFn(e[a-2],e[a])(b[a-2].addElse(b[a]));break;case 247:case 248:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.If(b[a],d.addLocationDataFn(e[a-2])(d.Block.wrap([b[a-2]])),{type:b[a-1],statement:!0}));break;case 251:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op("-",b[a]));break;case 252:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op("+",b[a]));break;case 253:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op("--", +b[a]));break;case 254:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op("++",b[a]));break;case 255:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op("--",b[a-1],null,!0));break;case 256:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Op("++",b[a-1],null,!0));break;case 257:this.$=d.addLocationDataFn(e[a-1],e[a])(new d.Existence(b[a-1]));break;case 258:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Op("+",b[a-2],b[a]));break;case 259:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Op("-",b[a-2],b[a]));break; +case 260:case 261:case 262:case 263:case 264:case 265:case 266:case 267:case 268:case 269:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Op(b[a-1],b[a-2],b[a]));break;case 270:e=d.addLocationDataFn(e[a-2],e[a]);b="!"===b[a-1].charAt(0)?(new d.Op(b[a-1].slice(1),b[a-2],b[a])).invert():new d.Op(b[a-1],b[a-2],b[a]);this.$=e(b);break;case 271:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Assign(b[a-2],b[a],b[a-1]));break;case 272:this.$=d.addLocationDataFn(e[a-4],e[a])(new d.Assign(b[a-4],b[a-1],b[a-3])); +break;case 273:this.$=d.addLocationDataFn(e[a-3],e[a])(new d.Assign(b[a-3],b[a],b[a-2]));break;case 274:this.$=d.addLocationDataFn(e[a-2],e[a])(new d.Extends(b[a-2],b[a]))}},table:[{1:[2,1],3:1,4:2,5:3,7:4,8:5,9:6,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:u,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k, +97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{1:[3]},{1:[2,2],6:qa},a(sa,[2,3]),a(sa,[2,6],{141:77,132:102,138:103,133:D,135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(sa,[2,7],{141:77,132:105,138:106,133:D,135:A,139:E,156:va}),a(sa,[2,8]),a(N,[2,14],{109:107,78:108,86:114,40:xa,41:xa,114:xa,82:ta,83:Na, +84:Fa,85:Ga,87:Ca,90:Pa,113:Ia}),a(N,[2,15],{86:114,109:117,78:118,82:ta,83:Na,84:Fa,85:Ga,87:Ca,90:Pa,113:Ia,114:xa}),a(N,[2,16]),a(N,[2,17]),a(N,[2,18]),a(N,[2,19]),a(N,[2,20]),a(N,[2,21]),a(N,[2,22]),a(N,[2,23]),a(N,[2,24]),a(N,[2,25]),a(N,[2,26]),a(Ea,[2,9]),a(Ea,[2,10]),a(Ea,[2,11]),a(Ea,[2,12]),a(Ea,[2,13]),a([1,6,32,42,131,133,135,139,156,163,164,165,166,167,168,169,170,171,172,173,174],Va,{15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,10:20,11:21,13:23,14:24,54:26, +47:27,79:28,80:29,81:30,111:31,66:33,77:40,154:41,132:43,136:44,138:45,74:53,75:54,37:55,43:57,33:70,60:71,141:77,39:80,7:120,8:122,12:b,28:ea,29:Ya,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,61:[1,119],62:z,63:l,67:c,68:w,92:m,95:k,97:K,105:P,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,137:q,149:ba,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}),a(Ba,Ha,{55:[1,124]}),a(Ba,[2,95]),a(Ba,[2,96]),a(Ba,[2,97]),a(Ba,[2,98]),a(t,[2,163]),a([6,31,65,70],p,{64:125,71:126,72:127,33:129,60:130, +74:131,75:132,34:g,73:d,92:m,118:wa,119:e}),{30:135,31:Da},{7:137,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C, +158:T,159:v,160:Y,161:S,162:M},{7:138,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}, +{7:139,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:140,8:122,10:20,11:21,12:b, +13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{15:142,16:143,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57, +44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:144,60:71,74:53,75:54,77:141,79:28,80:29,81:30,92:m,111:31,112:L,117:V,118:X,119:G,130:W},{15:142,16:143,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:144,60:71,74:53,75:54,77:145,79:28,80:29,81:30,92:m,111:31,112:L,117:V,118:X,119:G,130:W},a(Ta,ua,{96:[1,149],161:[1,146],162:[1,147],175:[1,148]}),a(N,[2,245],{151:[1,150]}),{30:151,31:Da},{30:152,31:Da},a(N,[2,209]),{30:153,31:Da},{7:154,8:122,10:20,11:21, +12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:[1,155],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Eb,[2,115],{47:27,79:28,80:29,81:30,111:31, +74:53,75:54,37:55,43:57,33:70,60:71,39:80,15:142,16:143,54:144,30:156,77:158,31:Da,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,92:m,96:[1,157],112:L,117:V,118:X,119:G,130:W}),{7:159,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P, +111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Ea,Za,{15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,10:20,11:21,13:23,14:24,54:26,47:27,79:28,80:29,81:30,111:31,66:33,77:40,154:41,132:43,136:44,138:45,74:53,75:54,37:55,43:57,33:70,60:71,141:77,39:80,8:122,7:160,12:b,28:ea,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,61:R,62:z,63:l,67:c,68:w, +92:m,95:k,97:K,105:P,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,137:q,149:ba,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}),a([1,6,31,32,42,70,94,131,133,135,139,156],[2,66]),{33:165,34:g,39:161,40:r,41:n,92:[1,164],98:162,99:163,104:Fb},{25:168,33:169,34:g,92:[1,167],95:k,103:[1,170],107:[1,171]},a(Ta,[2,92]),a(Ta,[2,93]),a(Ba,[2,40]),a(Ba,[2,41]),a(Ba,[2,42]),a(Ba,[2,43]),a(Ba,[2,44]),a(Ba,[2,45]),a(Ba,[2,46]),a(Ba,[2,47]),{4:172,5:3,7:4,8:5,9:6,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11, +20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:u,31:[1,173],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:174,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14, +23:15,24:16,25:17,26:18,27:19,28:ea,31:$a,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Ua,74:53,75:54,76:179,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,116:176,117:V,118:X,119:G,120:Gb,123:177,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Ba,[2,170]),a(Ba,[2,171],{35:181,36:Oa}),a([1,6,31,32,42,46,65,70,73,82, +83,84,85,87,89,90,94,113,115,120,122,131,133,134,135,139,140,156,159,160,163,164,165,166,167,168,169,170,171,172,173,174],[2,164],{110:183,114:sb}),{31:[2,69]},{31:[2,70]},a(La,[2,87]),a(La,[2,90]),{7:185,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K, +105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:186,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X, +119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:187,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43, +133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:189,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,30:188,31:Da,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44, +137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{33:194,34:g,60:195,74:196,75:197,80:190,92:m,118:wa,119:G,143:191,144:[1,192],145:193},{142:198,146:[1,199],147:[1,200],148:[1,201]},a([6,31,70,94],Hb,{39:80,93:202,56:203,57:204,59:205,11:206,37:207,33:208,35:209,60:210,34:g,36:Oa,38:h,40:r,41:n,62:z,118:wa}),a(Ib,[2,34]),a(Ib,[2,35]),a(Ba,[2,38]),{15:142,16:211,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:144,60:71, +74:53,75:54,77:212,79:28,80:29,81:30,92:m,111:31,112:L,117:V,118:X,119:G,130:W},a([1,6,29,31,32,40,41,42,55,58,65,70,73,82,83,84,85,87,89,90,94,96,102,113,114,115,120,122,131,133,134,135,139,140,146,147,148,156,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175],[2,32]),a(Jb,[2,36]),{4:213,5:3,7:4,8:5,9:6,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:u,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F, +50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(sa,[2,5],{7:4,8:5,9:6,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,10:20,11:21,13:23,14:24,54:26,47:27,79:28,80:29,81:30,111:31,66:33,77:40,154:41,132:43,136:44,138:45,74:53,75:54,37:55,43:57, +33:70,60:71,141:77,39:80,5:214,12:b,28:u,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,61:R,62:z,63:l,67:c,68:w,92:m,95:k,97:K,105:P,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,133:D,135:A,137:q,139:E,149:ba,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}),a(N,[2,257]),{7:215,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71, +61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:216,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w, +74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:217,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29, +81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:218,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31, +112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:219,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa, +129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:220,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A, +136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:221,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77, +149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:222,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T, +159:v,160:Y,161:S,162:M},{7:223,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:224, +8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:225,8:122,10:20,11:21,12:b,13:23, +14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:226,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11, +20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:227,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16, +25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:228,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70, +34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(N,[2,208]),a(N,[2,213]),{7:229,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g, +37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(N,[2,207]),a(N,[2,212]),{39:230,40:r,41:n,110:231,114:sb},a(La,[2,88]),a(Kb,[2,167]),{35:232,36:Oa},{35:233,36:Oa},a(La,[2,103],{35:234,36:Oa}),{35:235,36:Oa},a(La, +[2,104]),{7:237,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Lb,74:53,75:54,77:40,79:28,80:29,81:30,88:236,91:238,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,121:239,122:tb,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y, +161:S,162:M},{86:242,87:Ca,90:Pa},{110:243,114:sb},a(La,[2,89]),a(sa,[2,65],{15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,10:20,11:21,13:23,14:24,54:26,47:27,79:28,80:29,81:30,111:31,66:33,77:40,154:41,132:43,136:44,138:45,74:53,75:54,37:55,43:57,33:70,60:71,141:77,39:80,8:122,7:244,12:b,28:ea,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,61:R,62:z,63:l,67:c,68:w,92:m,95:k,97:K,105:P,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,133:Za,135:Za,139:Za,156:Za, +137:q,149:ba,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}),a(Ma,[2,28],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{7:245,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P, +111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{132:105,133:D,135:A,138:106,139:E,141:77,156:va},a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,163,164,165,166,167,168,169,170,171,172,173,174],Va,{15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,10:20,11:21,13:23,14:24,54:26,47:27,79:28,80:29,81:30,111:31,66:33,77:40,154:41,132:43,136:44, +138:45,74:53,75:54,37:55,43:57,33:70,60:71,141:77,39:80,7:120,8:122,12:b,28:ea,29:Ya,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,61:R,62:z,63:l,67:c,68:w,92:m,95:k,97:K,105:P,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,137:q,149:ba,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}),{6:[1,247],7:246,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:[1,248],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I, +49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a([6,31],Ja,{69:251,65:[1,249],70:Mb}),a(Sa,[2,74]),a(Sa,[2,78],{55:[1,253],73:[1,252]}),a(Sa,[2,81]),a(fb,[2,82]),a(fb,[2,83]),a(fb,[2,84]),a(fb,[2,85]),{35:181,36:Oa},{7:254,8:122,10:20,11:21,12:b,13:23,14:24,15:7, +16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:$a,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Ua,74:53,75:54,76:179,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,116:176,117:V,118:X,119:G,120:Gb,123:177,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(N,[2,68]),{4:256,5:3,7:4,8:5,9:6, +10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:u,32:[1,255],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a([1,6,31,32,42,65,70,73,89,94, +115,120,122,131,133,134,135,139,140,156,159,160,164,165,166,167,168,169,170,171,172,173,174],[2,249],{141:77,132:102,138:103,163:fa}),a(ab,[2,250],{141:77,132:102,138:103,163:fa,165:ga}),a(ab,[2,251],{141:77,132:102,138:103,163:fa,165:ga}),a(ab,[2,252],{141:77,132:102,138:103,163:fa,165:ga}),a(N,[2,253],{40:ua,41:ua,82:ua,83:ua,84:ua,85:ua,87:ua,90:ua,113:ua,114:ua}),a(Kb,xa,{109:107,78:108,86:114,82:ta,83:Na,84:Fa,85:Ga,87:Ca,90:Pa,113:Ia}),{78:118,82:ta,83:Na,84:Fa,85:Ga,86:114,87:Ca,90:Pa,109:117, +113:Ia,114:xa},a(Nb,Ha),a(N,[2,254],{40:ua,41:ua,82:ua,83:ua,84:ua,85:ua,87:ua,90:ua,113:ua,114:ua}),a(N,[2,255]),a(N,[2,256]),{6:[1,259],7:257,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:[1,258],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U, +130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:260,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44, +137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{30:261,31:Da,155:[1,262]},a(N,[2,192],{126:263,127:[1,264],128:[1,265]}),a(N,[2,206]),a(N,[2,214]),{31:[1,266],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},{150:267,152:268,153:gb},a(N,[2,116]),{7:270,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea, +33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Eb,[2,119],{30:271,31:Da,40:ua,41:ua,82:ua,83:ua,84:ua,85:ua,87:ua,90:ua,113:ua,114:ua,96:[1,272]}),a(Ma,[2,199],{141:77,132:102,138:103,159:ma,160:Z, +163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ea,bb,{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ea,[2,123]),{29:[1,273],70:[1,274]},{29:[1,275]},{31:hb,33:280,34:g,94:[1,276],100:277,101:278,103:Wa},a([29,70],[2,139]),{102:[1,282]},{31:ub,33:287,34:g,94:[1,283],103:cb,106:284,108:285},a(Ea,[2,143]),{55:[1,289]},{7:290,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11, +20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{29:[1,291]},{6:qa,131:[1,292]},{4:293,5:3,7:4,8:5,9:6,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9, +18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:u,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a([6,31,70,120],Ob,{141:77,132:102,138:103,121:294,73:[1,295],122:tb,133:D,135:A,139:E, +156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(vb,[2,173]),a([6,31,120],Ja,{69:296,70:ib}),a(Qa,[2,182]),{7:254,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:$a,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Ua,74:53,75:54,76:179,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31, +112:L,116:298,117:V,118:X,119:G,123:177,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Qa,[2,188]),a(Qa,[2,189]),a(Pb,[2,172]),a(Pb,[2,33]),a(t,[2,165]),{7:254,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:$a,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Ua, +74:53,75:54,76:179,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,115:[1,299],116:300,117:V,118:X,119:G,123:177,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{30:301,31:Da,132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},a(Qb,[2,202],{141:77,132:102,138:103,133:D,134:[1,302],135:A,139:E,159:ma,160:Z,163:fa,164:ia, +165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Qb,[2,204],{141:77,132:102,138:103,133:D,134:[1,303],135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(N,[2,210]),a(Xa,[2,211],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,156,159,160,163,164,165,166,167,168, +169,170,171,172,173,174],[2,215],{140:[1,304]}),a(jb,[2,218]),{33:194,34:g,60:195,74:196,75:197,92:m,118:wa,119:e,143:305,145:193},a(jb,[2,224],{70:[1,306]}),a(kb,[2,220]),a(kb,[2,221]),a(kb,[2,222]),a(kb,[2,223]),a(N,[2,217]),{7:307,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40, +79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:308,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k, +97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:309,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V, +118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(lb,Ja,{69:310,70:Rb}),a(Aa,[2,111]),a(Aa,[2,51],{58:[1,312]}),a(Sb,[2,60],{55:[1,313]}),a(Aa,[2,56]),a(Sb,[2,61]),a(wb,[2,57]),a(wb,[2,58]),a(wb,[2,59]),{46:[1,314],78:118,82:ta,83:Na,84:Fa,85:Ga,86:114,87:Ca,90:Pa,109:117,113:Ia,114:xa},a(Nb,ua),{6:qa,42:[1,315]},a(sa,[2,4]),a(Tb,[2,258],{141:77,132:102,138:103,163:fa,164:ia,165:ga}),a(Tb,[2,259],{141:77, +132:102,138:103,163:fa,164:ia,165:ga}),a(ab,[2,260],{141:77,132:102,138:103,163:fa,165:ga}),a(ab,[2,261],{141:77,132:102,138:103,163:fa,165:ga}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,166,167,168,169,170,171,172,173,174],[2,262],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,167,168,169,170,171,172,173],[2,263],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,174:da}), +a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,168,169,170,171,172,173],[2,264],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,169,170,171,172,173],[2,265],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,170,171,172,173],[2,266],{141:77,132:102,138:103,159:ma,160:Z, +163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,171,172,173],[2,267],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,172,173],[2,268],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134, +135,139,140,156,173],[2,269],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,174:da}),a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,134,135,139,140,156,167,168,169,170,171,172,173,174],[2,270],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja}),a(Xa,[2,248],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Xa,[2,247],{141:77,132:102, +138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(t,[2,160]),a(t,[2,161]),a(La,[2,99]),a(La,[2,100]),a(La,[2,101]),a(La,[2,102]),{89:[1,316]},{73:Lb,89:[2,107],121:317,122:tb,132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},{89:[2,108]},{7:318,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15, +24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,89:[2,181],92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Ub,[2,175]),a(Ub,Vb),a(La,[2,106]),a(t,[2,162]),a(sa,[2,64],{141:77,132:102,138:103,133:bb,135:bb,139:bb,156:bb, +159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ma,[2,29],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ma,[2,48],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{7:319,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g, +37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:320,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57, +44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{66:321,67:c,68:w},a(Ra,db,{72:127,33:129,60:130,74:131,75:132,71:322,34:g,73:d,92:m,118:wa,119:e}),{6:Wb,31:Xb},a(Sa,[2,79]),{7:325,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11, +20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Qa,Ob,{141:77,132:102,138:103,73:[1,326],133:D,135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga, +166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Yb,[2,30]),{6:qa,32:[1,327]},a(Ma,[2,271],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{7:328,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40, +79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:329,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k, +97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Ma,[2,274],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(N,[2,246]),{7:330,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27, +48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(N,[2,193],{127:[1,331]}),{30:332,31:Da},{30:335,31:Da,33:333,34:g,75:334,92:m},{150:336,152:268,153:gb},{32:[1,337],151:[1,338],152:339,153:gb},a(mb,[2,239]),{7:341,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8, +17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,124:340,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Zb,[2,117],{141:77,132:102,138:103,30:342,31:Da,133:D,135:A,139:E,159:ma, +160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(N,[2,120]),{7:343,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45, +139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{39:344,40:r,41:n},{92:[1,346],99:345,104:Fb},{39:347,40:r,41:n},{29:[1,348]},a(lb,Ja,{69:349,70:nb}),a(Aa,[2,130]),{31:hb,33:280,34:g,100:351,101:278,103:Wa},a(Aa,[2,135],{102:[1,352]}),a(Aa,[2,137],{102:[1,353]}),{33:354,34:g},a(Ea,[2,141]),a(lb,Ja,{69:355,70:xb}),a(Aa,[2,150]),{31:ub,33:287,34:g,103:cb,106:357,108:285},a(Aa,[2,155],{102:[1,358]}),a(Aa,[2,158],{102:[1,359]}),{6:[1,361],7:360,8:122,10:20,11:21,12:b,13:23,14:24, +15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:[1,362],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(yb,[2,147],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma, +160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{39:363,40:r,41:n},a(Ba,[2,200]),{6:qa,32:[1,364]},{7:365,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W, +132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a([12,28,34,38,40,41,44,45,48,49,50,51,52,53,61,62,63,67,68,92,95,97,105,112,117,118,119,125,129,130,133,135,137,139,149,155,157,158,159,160,161,162],Vb,{6:eb,31:eb,70:eb,120:eb}),{6:ob,31:pb,120:[1,366]},a([6,31,32,115,120],db,{15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,10:20,11:21,13:23,14:24,54:26,47:27,79:28,80:29,81:30,111:31,66:33,77:40,154:41,132:43, +136:44,138:45,74:53,75:54,37:55,43:57,33:70,60:71,141:77,39:80,8:122,76:179,7:254,123:369,12:b,28:ea,34:g,38:h,40:r,41:n,44:B,45:H,48:I,49:F,50:Q,51:x,52:J,53:O,61:R,62:z,63:l,67:c,68:w,73:Ua,92:m,95:k,97:K,105:P,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,133:D,135:A,137:q,139:E,149:ba,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}),a(Ra,Ja,{69:370,70:ib}),a(t,[2,168]),a([6,31,115],Ja,{69:371,70:ib}),a($b,[2,243]),{7:372,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14, +23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:373,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19, +28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:374,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h, +39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(jb,[2,219]),{33:194,34:g,60:195,74:196,75:197,92:m,118:wa,119:e,145:375},a([1,6,31,32,42,65,70,73,89,94,115,120,122,131,133,135,139,156],[2,226],{141:77,132:102,138:103,134:[1, +376],140:[1,377],159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(zb,[2,227],{141:77,132:102,138:103,134:[1,378],159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(zb,[2,233],{141:77,132:102,138:103,134:[1,379],159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{6:ac,31:bc,94:[1,380]},a(Ab,db,{39:80,57:204,59:205,11:206,37:207,33:208,35:209,60:210,56:383, +34:g,36:Oa,38:h,40:r,41:n,62:z,118:wa}),{7:384,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:[1,385],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v, +160:Y,161:S,162:M},{7:386,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:[1,387],33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M}, +a(Ba,[2,39]),a(Jb,[2,37]),a(La,[2,105]),{7:388,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,89:[2,179],92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v, +160:Y,161:S,162:M},{89:[2,180],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},a(Ma,[2,49],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{32:[1,389],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},{30:390,31:Da},a(Sa,[2,75]),{33:129, +34:g,60:130,71:391,72:127,73:d,74:131,75:132,92:m,118:wa,119:e},a(cc,p,{71:126,72:127,33:129,60:130,74:131,75:132,64:392,34:g,73:d,92:m,118:wa,119:e}),a(Sa,[2,80],{141:77,132:102,138:103,133:D,135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Qa,eb),a(Yb,[2,31]),{32:[1,393],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},a(Ma,[2,273], +{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{30:394,31:Da,132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},{30:395,31:Da},a(N,[2,194]),{30:396,31:Da},{30:397,31:Da},a(Bb,[2,198]),{32:[1,398],151:[1,399],152:339,153:gb},a(N,[2,237]),{30:400,31:Da},a(mb,[2,240]),{30:401,31:Da,70:[1,402]},a(dc,[2,190],{141:77,132:102,138:103,133:D, +135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(N,[2,118]),a(Zb,[2,121],{141:77,132:102,138:103,30:403,31:Da,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ea,[2,124]),{29:[1,404]},{31:hb,33:280,34:g,100:405,101:278,103:Wa},a(Ea,[2,125]),{39:406,40:r,41:n},{6:qb,31:rb,94:[1,407]},a(Ab,db,{33:280,101:410,34:g,103:Wa}),a(Ra,Ja,{69:411,70:nb}),{33:412,34:g}, +{33:413,34:g},{29:[2,140]},{6:Cb,31:Db,94:[1,414]},a(Ab,db,{33:287,108:417,34:g,103:cb}),a(Ra,Ja,{69:418,70:xb}),{33:419,34:g,103:[1,420]},{33:421,34:g},a(yb,[2,144],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{7:422,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q, +51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:423,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R, +62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Ea,[2,148]),{131:[1,424]},{120:[1,425],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},a(vb,[2,174]),{7:254,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9, +18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Ua,74:53,75:54,76:179,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,123:426,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:254,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11, +20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,31:$a,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,73:Ua,74:53,75:54,76:179,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,116:427,117:V,118:X,119:G,123:177,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Qa,[2,183]),{6:ob,31:pb,32:[1,428]},{6:ob,31:pb,115:[1,429]}, +a(Xa,[2,203],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Xa,[2,205],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Xa,[2,216],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(jb,[2,225]),{7:430,8:122,10:20,11:21, +12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:431,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9, +18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:432,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14, +23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:433,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19, +28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(vb,[2,109]),{11:206,33:208,34:g,35:209,36:Oa,37:207,38:h,39:80,40:r,41:n,56:434,57:204,59:205,60:210,62:z,118:wa},a(cc,Hb,{39:80,56:203,57:204, +59:205,11:206,37:207,33:208,35:209,60:210,93:435,34:g,36:Oa,38:h,40:r,41:n,62:z,118:wa}),a(Aa,[2,112]),a(Aa,[2,52],{141:77,132:102,138:103,133:D,135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{7:436,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l, +66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(Aa,[2,54],{141:77,132:102,138:103,133:D,135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{7:437,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18, +27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{89:[2,178],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na, +173:ra,174:da},a(N,[2,50]),a(N,[2,67]),a(Sa,[2,76]),a(Ra,Ja,{69:438,70:Mb}),a(N,[2,272]),a($b,[2,244]),a(N,[2,195]),a(Bb,[2,196]),a(Bb,[2,197]),a(N,[2,235]),{30:439,31:Da},{32:[1,440]},a(mb,[2,241],{6:[1,441]}),{7:442,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30, +92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},a(N,[2,122]),{39:443,40:r,41:n},a(lb,Ja,{69:444,70:nb}),a(Ea,[2,126]),{29:[1,445]},{33:280,34:g,101:446,103:Wa},{31:hb,33:280,34:g,100:447,101:278,103:Wa},a(Aa,[2,131]),{6:qb,31:rb,32:[1,448]},a(Aa,[2,136]),a(Aa,[2,138]),a(Ea,[2,142],{29:[1,449]}),{33:287,34:g,103:cb,108:450},{31:ub,33:287,34:g,103:cb,106:451,108:285}, +a(Aa,[2,151]),{6:Cb,31:Db,32:[1,452]},a(Aa,[2,156]),a(Aa,[2,157]),a(Aa,[2,159]),a(yb,[2,145],{141:77,132:102,138:103,133:D,135:A,139:E,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),{32:[1,453],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},a(Ba,[2,201]),a(Ba,[2,177]),a(Qa,[2,184]),a(Ra,Ja,{69:454,70:ib}),a(Qa,[2,185]),a(t,[2,169]),a([1,6,31,32,42, +65,70,73,89,94,115,120,122,131,133,134,135,139,156],[2,228],{141:77,132:102,138:103,140:[1,455],159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(zb,[2,230],{141:77,132:102,138:103,134:[1,456],159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ma,[2,229],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ma,[2,234],{141:77,132:102, +138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Aa,[2,113]),a(Ra,Ja,{69:457,70:Rb}),{32:[1,458],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},{32:[1,459],132:102,133:D,135:A,138:103,139:E,141:77,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da},{6:Wb,31:Xb,32:[1,460]},{32:[1,461]},a(N, +[2,238]),a(mb,[2,242]),a(dc,[2,191],{141:77,132:102,138:103,133:D,135:A,139:E,156:za,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ea,[2,128]),{6:qb,31:rb,94:[1,462]},{39:463,40:r,41:n},a(Aa,[2,132]),a(Ra,Ja,{69:464,70:nb}),a(Aa,[2,133]),{39:465,40:r,41:n},a(Aa,[2,152]),a(Ra,Ja,{69:466,70:xb}),a(Aa,[2,153]),a(Ea,[2,146]),{6:ob,31:pb,32:[1,467]},{7:468,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16, +25:17,26:18,27:19,28:ea,33:70,34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{7:469,8:122,10:20,11:21,12:b,13:23,14:24,15:7,16:8,17:9,18:10,19:11,20:12,21:13,22:14,23:15,24:16,25:17,26:18,27:19,28:ea,33:70, +34:g,37:55,38:h,39:80,40:r,41:n,43:57,44:B,45:H,47:27,48:I,49:F,50:Q,51:x,52:J,53:O,54:26,60:71,61:R,62:z,63:l,66:33,67:c,68:w,74:53,75:54,77:40,79:28,80:29,81:30,92:m,95:k,97:K,105:P,111:31,112:L,117:V,118:X,119:G,125:aa,129:U,130:W,132:43,133:D,135:A,136:44,137:q,138:45,139:E,141:77,149:ba,154:41,155:ca,157:C,158:T,159:v,160:Y,161:S,162:M},{6:ac,31:bc,32:[1,470]},a(Aa,[2,53]),a(Aa,[2,55]),a(Sa,[2,77]),a(N,[2,236]),{29:[1,471]},a(Ea,[2,127]),{6:qb,31:rb,32:[1,472]},a(Ea,[2,149]),{6:Cb,31:Db,32:[1, +473]},a(Qa,[2,186]),a(Ma,[2,231],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Ma,[2,232],{141:77,132:102,138:103,159:ma,160:Z,163:fa,164:ia,165:ga,166:ja,167:la,168:oa,169:pa,170:ha,171:ka,172:na,173:ra,174:da}),a(Aa,[2,114]),{39:474,40:r,41:n},a(Aa,[2,134]),a(Aa,[2,154]),a(Ea,[2,129])],defaultActions:{68:[2,69],69:[2,70],238:[2,108],354:[2,140]},parseError:function(a,d){if(d.recoverable)this.trace(a);else{var e=function(a, +d){this.message=a;this.hash=d};e.prototype=Error;throw new e(a,d);}},parse:function(a){var d=[0],e=[null],b=[],p=this.table,t="",wa=0,c=0,g=0,Da=b.slice.call(arguments,1),k=Object.create(this.lexer),h={};for(f in this.yy)Object.prototype.hasOwnProperty.call(this.yy,f)&&(h[f]=this.yy[f]);k.setInput(a,h);h.lexer=k;h.parser=this;"undefined"==typeof k.yylloc&&(k.yylloc={});var f=k.yylloc;b.push(f);var l=k.options&&k.options.ranges;this.parseError="function"===typeof h.parseError?h.parseError:Object.getPrototypeOf(this).parseError; +for(var m,Ta,Ha,n,ua={},y,w;;){Ha=d[d.length-1];if(this.defaultActions[Ha])n=this.defaultActions[Ha];else{if(null===m||"undefined"==typeof m)m=k.lex()||1,"number"!==typeof m&&(m=this.symbols_[m]||m);n=p[Ha]&&p[Ha][m]}if("undefined"===typeof n||!n.length||!n[0]){w=[];for(y in p[Ha])this.terminals_[y]&&2=ta?this.wrapInBraces(d):d};b.prototype.compileRoot=function(a){var d,b;a.indent=a.bare?"":Ca;a.level=N;this.spaced=!0;a.scope=new xa(null,this,null,null!=(b=a.referencedVars)?b:[]);var e=a.locals||[];b=0;for(d=e.length;b=Fa?this.wrapInBraces(d): +d};return b}(w);f.StringLiteral=D=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);return b}(z);f.RegexLiteral=X=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);return b}(z);f.PassthroughLiteral=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);return b}(z);f.IdentifierLiteral=x=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);b.prototype.isAssignable=ha; +return b}(z);f.PropertyName=L=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);b.prototype.isAssignable=ha;return b}(z);f.StatementLiteral=W=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);b.prototype.isStatement=ha;b.prototype.makeReturn=na;b.prototype.jumps=function(a){if("break"===this.value&&!(null!=a&&a.loop||null!=a&&a.block)||"continue"===this.value&&(null==a||!a.loop))return this};b.prototype.compileNode=function(a){return[this.makeCode(""+ +this.tab+this.value+";")]};return b}(z);f.ThisLiteral=E=function(a){function b(){b.__super__.constructor.call(this,"this")}v(b,a);b.prototype.compileNode=function(a){var d;a=null!=(d=a.scope.method)&&d.bound?a.scope.method.context:this.value;return[this.makeCode(a)]};return b}(z);f.UndefinedLiteral=ca=function(a){function b(){b.__super__.constructor.call(this,"undefined")}v(b,a);b.prototype.compileNode=function(a){return[this.makeCode(a.level>=Ga?"(void 0)":"void 0")]};return b}(z);f.NullLiteral= +c=function(a){function b(){b.__super__.constructor.call(this,"null")}v(b,a);return b}(z);f.BooleanLiteral=b=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);return b}(z);f.Return=G=function(a){function b(a){this.expression=a}v(b,a);b.prototype.children=["expression"];b.prototype.isStatement=ha;b.prototype.makeReturn=na;b.prototype.jumps=na;b.prototype.compileToFragments=function(a,d){var p;var e=null!=(p=this.expression)?p.makeReturn():void 0;return!e||e instanceof +b?b.__super__.compileToFragments.call(this,a,d):e.compileToFragments(a,d)};b.prototype.compileNode=function(a){var b=[];b.push(this.makeCode(this.tab+("return"+(this.expression?" ":""))));this.expression&&(b=b.concat(this.expression.compileToFragments(a,Ka)));b.push(this.makeCode(";"));return b};return b}(sa);f.YieldReturn=T=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);b.prototype.compileNode=function(a){null==a.scope.parent&&this.error("yield can only occur inside functions"); +return b.__super__.compileNode.apply(this,arguments)};return b}(G);f.Value=C=function(a){function t(a,b,wa){if(!b&&a instanceof t)return a;this.base=a;this.properties=b||[];wa&&(this[wa]=!0);return this}v(t,a);t.prototype.children=["base","properties"];t.prototype.add=function(a){this.properties=this.properties.concat(a);return this};t.prototype.hasProperties=function(){return!!this.properties.length};t.prototype.bareLiteral=function(a){return!this.properties.length&&this.base instanceof a};t.prototype.isArray= +function(){return this.bareLiteral(q)};t.prototype.isRange=function(){return this.bareLiteral(V)};t.prototype.isComplex=function(){return this.hasProperties()||this.base.isComplex()};t.prototype.isAssignable=function(){return this.hasProperties()||this.base.isAssignable()};t.prototype.isNumber=function(){return this.bareLiteral(w)};t.prototype.isString=function(){return this.bareLiteral(D)};t.prototype.isRegex=function(){return this.bareLiteral(X)};t.prototype.isUndefined=function(){return this.bareLiteral(ca)}; +t.prototype.isNull=function(){return this.bareLiteral(c)};t.prototype.isBoolean=function(){return this.bareLiteral(b)};t.prototype.isAtomic=function(){var a;var b=this.properties.concat(this.base);var wa=0;for(a=b.length;wathis.properties.length&&!this.base.isComplex()&&(null==p||!p.isComplex()))return[this,this];b=new t(this.base,this.properties.slice(0,-1));if(b.isComplex()){var e=new x(a.scope.freeVariable("base"));b=new t(new P(new y(e, +b)))}if(!p)return[b,e];if(p.isComplex()){var c=new x(a.scope.freeVariable("name"));p=new R(new y(c,p.index));c=new R(c)}return[b.add(p),new t(e||b.base,[c||p])]};t.prototype.compileNode=function(a){var b;this.base.front=this.front;var p=this.properties;var e=this.base.compileToFragments(a,p.length?Ga:null);p.length&&Pa.test(da(e))&&e.push(this.makeCode("."));var t=0;for(b=p.length;t=Math.abs(this.fromNum-this.toNum)){var c=function(){e=[];for(var a=p=this.fromNum,b=this.toNum;p<=b?a<=b:a>=b;p<=b?a++:a--)e.push(a);return e}.apply(this);this.exclusive&&c.pop();return[this.makeCode("["+c.join(", ")+"]")]}var t=this.tab+Ca;var f=a.scope.freeVariable("i",{single:!0});var g=a.scope.freeVariable("results");var k="\n"+t+g+" \x3d [];";if(b)a.index=f,b=da(this.compileNode(a));else{var h= +f+" \x3d "+this.fromC+(this.toC!==this.toVar?", "+this.toC:"");b=this.fromVar+" \x3c\x3d "+this.toVar;b="var "+h+"; "+b+" ? "+f+" \x3c"+this.equals+" "+this.toVar+" : "+f+" \x3e"+this.equals+" "+this.toVar+"; "+b+" ? "+f+"++ : "+f+"--"}f="{ "+g+".push("+f+"); }\n"+t+"return "+g+";\n"+a.indent;a=function(a){return null!=a?a.contains(Va):void 0};if(a(this.from)||a(this.to))c=", arguments";return[this.makeCode("(function() {"+k+"\n"+t+"for ("+b+")"+f+"}).apply(this"+(null!=c?c:"")+")")]};return b}(sa); +f.Slice=aa=function(a){function b(a){this.range=a;b.__super__.constructor.call(this)}v(b,a);b.prototype.children=["range"];b.prototype.compileNode=function(a){var b=this.range;var p=b.to;var e=(b=b.from)&&b.compileToFragments(a,Ka)||[this.makeCode("0")];if(p){b=p.compileToFragments(a,Ka);var c=da(b);if(this.range.exclusive||-1!==+c)var t=", "+(this.range.exclusive?c:p.isNumber()?""+(+c+1):(b=p.compileToFragments(a,Ga),"+"+da(b)+" + 1 || 9e9"))}return[this.makeCode(".slice("+da(e)+(t||"")+")")]};return b}(sa); +f.Obj=m=function(a){function b(a,b){this.generated=null!=b?b:!1;this.objects=this.properties=a||[]}v(b,a);b.prototype.children=["properties"];b.prototype.compileNode=function(a){var b,p,e;var c=this.properties;if(this.generated){var t=0;for(b=c.length;t= +Fa?this.wrapInBraces(t):t}var h=g[0];1===e&&h instanceof H&&h.error("Destructuring assignment has no target");var m=this.variable.isObject();if(p&&1===e&&!(h instanceof U)){var l=null;if(h instanceof b&&"object"===h.context){t=h;var n=t.variable;var q=n.base;h=t.value;h instanceof b&&(l=h.value,h=h.variable)}else h instanceof b&&(l=h.value,h=h.variable),q=m?h["this"]?h.properties[0].name:new L(h.unwrap().value):new w(0);var r=q.unwrap()instanceof L;f=new C(f);f.properties.push(new (r?qa:R)(q));(c= +za(h.unwrap().value))&&h.error(c);l&&(f=new k("?",f,l));return(new b(h,f,null,{param:this.param})).compileToFragments(a,N)}var v=f.compileToFragments(a,ta);var y=da(v);t=[];n=!1;f.unwrap()instanceof x&&!this.variable.assigns(y)||(t.push([this.makeCode((l=a.scope.freeVariable("ref"))+" \x3d ")].concat(M.call(v))),v=[this.makeCode(l)],y=l);l=f=0;for(d=g.length;fN?this.wrapInBraces(e):e};return b}(sa);f.Code=h=function(b){function c(b,d,c){this.params=b||[];this.body=d||new a;this.bound="boundfunc"===c;this.isGenerator=!!this.body.contains(function(a){return a instanceof k&&a.isYield()|| +a instanceof T})}v(c,b);c.prototype.children=["params","body"];c.prototype.isStatement=function(){return!!this.ctor};c.prototype.jumps=ka;c.prototype.makeScope=function(a){return new xa(a,this.body,this)};c.prototype.compileNode=function(b){var d,f,e,g;this.bound&&null!=(d=b.scope.method)&&d.bound&&(this.context=b.scope.method.context);if(this.bound&&!this.context)return this.context="_this",d=new c([new K(new x(this.context))],new a([this])),d=new ya(d,[new E]),d.updateLocationDataIfMissing(this.locationData), +d.compileNode(b);b.scope=la(b,"classScope")||this.makeScope(b.scope);b.scope.shared=la(b,"sharedScope");b.indent+=Ca;delete b.bare;delete b.isExistentialEquals;d=[];var p=[];var h=this.params;var t=0;for(e=h.length;t=Ga?this.wrapInBraces(p):p};c.prototype.eachParamName=function(a){var b;var c=this.params;var e=[];var f=0;for(b=c.length;f=d.length)return[];if(1===d.length)return e=d[0],d=e.compileToFragments(a,ta),c?d:[].concat(e.makeCode(Ia("slice",a)+".call("),d,e.makeCode(")"));c=d.slice(f);var h=g=0;for(p=c.length;g< +p;h=++g){e=c[h];var k=e.compileToFragments(a,ta);c[h]=e instanceof b?[].concat(e.makeCode(Ia("slice",a)+".call("),k,e.makeCode(")")):[].concat(e.makeCode("["),k,e.makeCode("]"))}if(0===f)return e=d[0],a=e.joinFragmentArrays(c.slice(1),", "),c[0].concat(e.makeCode(".concat("),a,e.makeCode(")"));g=d.slice(0,f);p=[];k=0;for(h=g.length;k=Ga)return(new P(this)).compileToFragments(a);var f="+"===c||"-"===c;("new"===c||"typeof"===c||"delete"===c||f&&this.first instanceof b&&this.first.operator===c)&&d.push([this.makeCode(" ")]);if(f&&this.first instanceof b||"new"===c&&this.first.isStatement(a))this.first=new P(this.first);d.push(this.first.compileToFragments(a,Fa));this.flip&&d.reverse();return this.joinFragmentArrays(d,"")};b.prototype.compileYield=function(a){var b; +var d=[];var c=this.operator;null==a.scope.parent&&this.error("yield can only occur inside functions");0<=S.call(Object.keys(this.first),"expression")&&!(this.first instanceof ba)?null!=this.first.expression&&d.push(this.first.expression.compileToFragments(a,Fa)):(a.level>=Ka&&d.push([this.makeCode("(")]),d.push([this.makeCode(c)]),""!==(null!=(b=this.first.base)?b.value:void 0)&&d.push([this.makeCode(" ")]),d.push(this.first.compileToFragments(a,Fa)),a.level>=Ka&&d.push([this.makeCode(")")]));return this.joinFragmentArrays(d, +"")};b.prototype.compilePower=function(a){var b=new C(new x("Math"),[new qa(new L("pow"))]);return(new ya(b,[this.first,this.second])).compileToFragments(a)};b.prototype.compileFloorDivision=function(a){var d=new C(new x("Math"),[new qa(new L("floor"))]);var c=this.second.isComplex()?new P(this.second):this.second;c=new b("/",this.first,c);return(new ya(d,[c])).compileToFragments(a)};b.prototype.compileModulo=function(a){var b=new C(new z(Ia("modulo",a)));return(new ya(b,[this.first,this.second])).compileToFragments(a)}; +b.prototype.toString=function(a){return b.__super__.toString.call(this,a,this.constructor.name+" "+this.operator)};return b}(sa);f.In=O=function(a){function b(a,b){this.object=a;this.array=b}v(b,a);b.prototype.children=["object","array"];b.prototype.invert=ra;b.prototype.compileNode=function(a){var b;if(this.array instanceof C&&this.array.isArray()&&this.array.base.objects.length){var c=this.array.base.objects;var e=0;for(b=c.length;e=c.length)?c:this.wrapInBraces(c)};return b}(sa); +f.StringWithInterpolations=A=function(a){function b(){return b.__super__.constructor.apply(this,arguments)}v(b,a);b.prototype.compileNode=function(a){var d;if(!a.inTaggedTemplateCall)return b.__super__.compileNode.apply(this,arguments);var c=this.body.unwrap();var e=[];c.traverseChildren(!1,function(a){if(a instanceof D)e.push(a);else if(a instanceof P)return e.push(a),!1;return!0});c=[];c.push(this.makeCode("`"));var f=0;for(d=e.length;fh,this.step&&null!=h&&e||(d=n.freeVariable("len")),K=""+t+f+" \x3d 0, "+d+" \x3d "+A+".length",w=""+t+f+" \x3d "+A+".length - 1",d=f+" \x3c "+d,n=f+" \x3e\x3d 0",this.step?(null!=h?e&&(d= +n,K=w):(d=r+" \x3e 0 ? "+d+" : "+n,K="("+r+" \x3e 0 ? ("+K+") : "+w+")"),f=f+" +\x3d "+r):f=""+(q!==f?"++"+f:f+"++"),K=[this.makeCode(K+"; "+d+"; "+t+f)])}if(this.returns){var B=""+this.tab+c+" \x3d [];\n";var V="\n"+this.tab+"return "+c+";";l.makeReturn(c)}this.guard&&(1=Na?this.wrapInBraces(e):e};c.prototype.unfoldSoak=function(){return this.soak&&this};return c}(sa);var gc={extend:function(a){return"function(child, parent) { for (var key in parent) { if ("+Ia("hasProp",a)+".call(parent, key)) child[key] \x3d parent[key]; } function ctor() { this.constructor \x3d child; } ctor.prototype \x3d parent.prototype; child.prototype \x3d new ctor(); child.__super__ \x3d parent.prototype; return child; }"},bind:function(){return"function(fn, me){ return function(){ return fn.apply(me, arguments); }; }"}, +indexOf:function(){return"[].indexOf || function(item) { for (var i \x3d 0, l \x3d this.length; i \x3c l; i++) { if (i in this \x26\x26 this[i] \x3d\x3d\x3d item) return i; } return -1; }"},modulo:function(){return"function(a, b) { return (+a % (b \x3d +b) + b) % b; }"},hasProp:function(){return"{}.hasOwnProperty"},slice:function(){return"[].slice"}};var N=1;var Ka=2;var ta=3;var Na=4;var Fa=5;var Ga=6;var Ca=" ";var Pa=/^[+-]?\d+$/;var Ia=function(a,b){var c=b.scope.root;if(a in c.utilities)return c.utilities[a]; +var d=c.freeVariable(a);c.assign(d,gc[a](b));return c.utilities[a]=d};var Ea=function(a,b){a=a.replace(/\n/g,"$\x26"+b);return a.replace(/\s+$/,"")};var Va=function(a){return a instanceof x&&"arguments"===a.value};var ea=function(a){return a instanceof E||a instanceof h&&a.bound||a instanceof va};var Ya=function(a){return a.isComplex()||("function"===typeof a.isAssignable?a.isAssignable():void 0)};var Ba=function(a,b,c){if(a=b[c].unfoldSoak(a))return b[c]=a.body,a.body=new C(b),a}}).call(this);return f}(); +u["./sourcemap"]=function(){var f={};(function(){var u=function(){function f(f){this.line=f;this.columns=[]}f.prototype.add=function(f,a,b){var q=a[0];a=a[1];null==b&&(b={});if(!this.columns[f]||!b.noReplace)return this.columns[f]={line:this.line,column:f,sourceLine:q,sourceColumn:a}};f.prototype.sourceLocation=function(f){for(var a;!((a=this.columns[f])||0>=f);)f--;return a&&[a.sourceLine,a.sourceColumn]};return f}();f=function(){function f(){this.lines=[]}f.prototype.add=function(f,a,b){var q;null== +b&&(b={});var g=a[0];a=a[1];return((q=this.lines)[g]||(q[g]=new u(g))).add(a,f,b)};f.prototype.sourceLocation=function(f){var a;var b=f[0];for(f=f[1];!((a=this.lines[b])||0>=b);)b--;return a&&a.sourceLocation(f)};f.prototype.generate=function(f,a){var b,q,g,h,r,n,u;null==f&&(f={});null==a&&(a=null);var y=g=q=u=0;var I=!1;var F="";var Q=this.lines;var x=b=0;for(h=Q.length;bf?1:0);a||!b;)f=a&31,(a>>=5)&&(f|=32),b+=this.encodeBase64(f);return b};f.prototype.encodeBase64=function(f){var a;if(!(a= +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[f]))throw Error("Cannot Base64 encode value: "+f);return a};return f}()}).call(this);return f}();u["./coffee-script"]=function(){var f={};(function(){var qa,q,y={}.hasOwnProperty;var a=u("fs");var b=u("vm");var ya=u("path");var g=u("./lexer").Lexer;var h=u("./parser").parser;var r=u("./helpers");var n=u("./sourcemap");var B=u("../../package.json");f.VERSION=B.version;f.FILE_EXTENSIONS=[".coffee",".litcoffee",".coffee.md"];f.helpers= +r;var H=function(a){switch(!1){case "function"!==typeof Buffer:return(new Buffer(a)).toString("base64");case "function"!==typeof btoa:return btoa(encodeURIComponent(a).replace(/%([0-9A-F]{2})/g,function(a,b){return String.fromCharCode("0x"+b)}));default:throw Error("Unable to base64 encode inline sourcemap.");}};B=function(a){return function(b,f){null==f&&(f={});try{return a.call(this,b,f)}catch(m){if("string"!==typeof b)throw m;throw r.updateSyntaxError(m,b,f.filename);}}};var I={};var F={};f.compile= +qa=B(function(a,b){var c,f,g,l;var q=r.extend;b=q({},b);var u=b.sourceMap||b.inlineMap||null==b.filename;q=b.filename||"\x3canonymous\x3e";I[q]=a;u&&(g=new n);var x=O.tokenize(a,b);var y=b;var G=[];var z=0;for(c=x.length;z + + + \ No newline at end of file diff --git a/trackers.txt b/trackers.txt new file mode 100644 index 00000000..a42f8ca4 --- /dev/null +++ b/trackers.txt @@ -0,0 +1,142 @@ +udp://tracker.opentrackr.org:1337/announce +udp://explodie.org:6969/announce +udp://open.stealth.si:80/announce +http://tracker.ipv6tracker.ru:80/announce +udp://tracker.birkenwald.de:6969/announce +udp://tracker.moeking.me:6969/announce +http://tracker.bt4g.com:2095/announce +https://tracker.nanoha.org:443/announce +http://tracker.files.fm:6969/announce +http://open.acgnxtracker.com:80/announce +udp://tracker.army:6969/announce +udp://fe.dealclub.de:6969/announce +udp://tracker.leech.ie:1337/announce +udp://tracker.altrosky.nl:6969/announce +https://tracker.cyber-hub.net:443/announce +https://tracker.lilithraws.cf:443/announce +http://bt.okmp3.ru:2710/announce +udp://vibe.sleepyinternetfun.xyz:1738/announce +udp://open.publictracker.xyz:6969/announce +udp://tracker.bitsearch.to:1337/announce +udp://tracker.pomf.se:80/announce +https://tr.burnabyhighstar.com:443/announce +https://tr.abiir.top:443/announce +udp://open.free-tracker.ga:6969/announce +http://i-p-v-6.tk:6969/announce +http://open-v6.demonoid.ch:6969/announce +udp://aarsen.me:6969/announce +udp://htz3.noho.st:6969/announce +udp://uploads.gamecoast.net:6969/announce +udp://mail.zasaonsk.ga:6969/announce +udp://tracker.joybomb.tw:6969/announce +udp://tracker.jonaslsa.com:6969/announce +udp://leefafa.tk:6969/announce +udp://carr.codes:6969/announce +https://tr.fuckbitcoin.xyz:443/announce +udp://tracker.cubonegro.xyz:6969/announce +udp://tracker.skynetcloud.site:6969/announce +http://tracker4.itzmx.com:2710/announce +https://tracker.lilithraws.org:443/announce +udp://tracker.novaopcj.eu.org:6969/announce +udp://exodus.desync.com:6969/announce +http://t.acg.rip:6699/announce +udp://tracker2.dler.com:80/announce +udp://6ahddutb1ucc3cp.ru:6969/announce +udp://tracker.blacksparrowmedia.net:6969/announce +http://fxtt.ru:80/announce +udp://tracker.auctor.tv:6969/announce +udp://torrentclub.space:6969/announce +udp://zecircle.xyz:6969/announce +udp://psyco.fr:6969/announce +udp://fh2.cmp-gaming.com:6969/announce +udp://new-line.net:6969/announce +udp://torrents.artixlinux.org:6969/announce +udp://bt.ktrackers.com:6666/announce +udp://static.54.161.216.95.clients.your-server.de:6969/announce +udp://cpe-104-34-3-152.socal.res.rr.com:6969/announce +http://t.overflow.biz:6969/announce +udp://tracker1.myporn.club:9337/announce +udp://moonburrow.club:6969/announce +udp://tracker.artixlinux.org:6969/announce +https://t1.hloli.org:443/announce +udp://bt1.archive.org:6969/announce +udp://tracker.theoks.net:6969/announce +udp://tracker.4.babico.name.tr:3131/announce +udp://buddyfly.top:6969/announce +udp://ipv6.tracker.harry.lu:80/announce +udp://public.publictracker.xyz:6969/announce +udp://mail.artixlinux.org:6969/announce +udp://v1046920.hosted-by-vdsina.ru:6969/announce +udp://tracker.cyberia.is:6969/announce +udp://tracker.beeimg.com:6969/announce +udp://creative.7o7.cx:6969/announce +udp://open.dstud.io:6969/announce +udp://laze.cc:6969/announce +udp://download.nerocloud.me:6969/announce +udp://cutscloud.duckdns.org:6969/announce +https://tracker.jiesen.life:8443/announce +udp://jutone.com:6969/announce +udp://wepzone.net:6969/announce +udp://ipv4.tracker.harry.lu:80/announce +udp://tracker.tcp.exchange:6969/announce +udp://f1sh.de:6969/announce +udp://movies.zsw.ca:6969/announce +https://tracker1.ctix.cn:443/announce +udp://sanincode.com:6969/announce +udp://www.torrent.eu.org:451/announce +udp://open.4ever.tk:6969/announce +https://tracker2.ctix.cn:443/announce +udp://bt2.archive.org:6969/announce +http://t.nyaatracker.com:80/announce +udp://yahor.ftp.sh:6969/announce +udp://tracker.openbtba.com:6969/announce +udp://tracker.dler.com:6969/announce +udp://tracker-udp.gbitt.info:80/announce +udp://tracker.srv00.com:6969/announce +udp://tracker.pimpmyworld.to:6969/announce +http://tracker.gbitt.info:80/announce +udp://tracker6.lelux.fi:6969/announce +http://tracker.vrpnet.org:6969/announce +http://00.xxtor.com:443/announce +http://vps02.net.orel.ru:80/announce +udp://tracker.yangxiaoguozi.cn:6969/announce +udp://rep-art.ynh.fr:6969/announce +https://tracker.imgoingto.icu:443/announce +udp://mirror.aptus.co.tz:6969/announce +udp://tracker.lelux.fi:6969/announce +udp://tracker.torrent.eu.org:451/announce +udp://admin.52ywp.com:6969/announce +udp://thouvenin.cloud:6969/announce +http://vps-dd0a0715.vps.ovh.net:6969/announce +udp://bubu.mapfactor.com:6969/announce +udp://94-227-232-84.access.telenet.be:6969/announce +udp://epider.me:6969/announce +udp://camera.lei001.com:6969/announce +udp://tamas3.ynh.fr:6969/announce +https://tracker.tamersunion.org:443/announce +udp://ftp.pet:2710/announce +udp://p4p.arenabg.com:1337/announce +http://tracker.mywaifu.best:6969/announce +udp://tracker.monitorit4.me:6969/announce +udp://ipv6.tracker.monitorit4.me:6969/announce +zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441 +zero://2kcb2fqesyaevc4lntogupa4mkdssth2ypfwczd2ov5a3zo6ytwwbayd.onion:15441 +zero://5vczpwawviukvd7grfhsfxp7a6huz77hlis4fstjkym5kmf4pu7i7myd.onion:15441 +zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441 +zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441 +zero://tl74auz4tyqv4bieeclmyoe4uwtoc2dj7fdqv4nc4gl5j2bwg2r26bqd.onion:15441 +zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441 +zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441 +zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441 +zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441 +zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441 +zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445 +zero://hb6ozikfiaafeuqvgseiik4r46szbpjfu66l67wjinnyv6dtopuwhtqd.onion:15445 +zero://qn65si4gtcwdiliq7vzrwu62qrweoxb6tx2cchwslaervj6szuje66qd.onion:26117 +zero://s3j2s5pjdfesbsmaqx6alsumaxxdxibmhv4eukmqpv3vqj6f627qx5yd.onion:15441 +zero://agufghdtniyfwty3wk55drxxwj2zxgzzo7dbrtje73gmvcpxy4ngs4ad.onion:15441 +zero://kgsvasoakvj4gnjiy7zemu34l3hq46dn5eauqkn76jpowmilci5t2vqd.onion:15445 +zero://dslesoe72bdfwfu4cfqa2wpd4hr3fhlu4zv6mfsjju5xlpmssouv36qd.onion:15441 +zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441 +zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441 +zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441 \ No newline at end of file diff --git a/update.py b/update.py new file mode 100644 index 00000000..cf9898f9 --- /dev/null +++ b/update.py @@ -0,0 +1,120 @@ +import os +import sys +import json +import re +import shutil + + +def update(): + from Config import config + config.parse(silent=True) + + if getattr(sys, 'source_update_dir', False): + if not os.path.isdir(sys.source_update_dir): + os.makedirs(sys.source_update_dir) + source_path = sys.source_update_dir.rstrip("/") + else: + source_path = os.getcwd().rstrip("/") + + if config.dist_type.startswith("bundle_linux"): + runtime_path = os.path.normpath(os.path.dirname(sys.executable) + "/../..") + else: + runtime_path = os.path.dirname(sys.executable) + + updatesite_path = config.data_dir + "/" + config.updatesite + + sites_json = json.load(open(config.data_dir + "/sites.json")) + updatesite_bad_files = sites_json.get(config.updatesite, {}).get("cache", {}).get("bad_files", {}) + print( + "Update site path: %s, bad_files: %s, source path: %s, runtime path: %s, dist type: %s" % + (updatesite_path, len(updatesite_bad_files), source_path, runtime_path, config.dist_type) + ) + + updatesite_content_json = json.load(open(updatesite_path + "/content.json")) + inner_paths = list(updatesite_content_json.get("files", {}).keys()) + inner_paths += list(updatesite_content_json.get("files_optional", {}).keys()) + + # Keep file only in ZeroNet directory + inner_paths = [inner_path for inner_path in inner_paths if re.match("^(core|bundle)", inner_path)] + + # Checking plugins + plugins_enabled = [] + plugins_disabled = [] + if os.path.isdir("%s/plugins" % source_path): + for dir in os.listdir("%s/plugins" % source_path): + if dir.startswith("disabled-"): + plugins_disabled.append(dir.replace("disabled-", "")) + else: + plugins_enabled.append(dir) + print("Plugins enabled:", plugins_enabled, "disabled:", plugins_disabled) + + update_paths = {} + + for inner_path in inner_paths: + if ".." in inner_path: + continue + inner_path = inner_path.replace("\\", "/").strip("/") # Make sure we have unix path + print(".", end=" ") + if inner_path.startswith("core"): + dest_path = source_path + "/" + re.sub("^core/", "", inner_path) + elif inner_path.startswith(config.dist_type): + dest_path = runtime_path + "/" + re.sub("^bundle[^/]+/", "", inner_path) + else: + continue + + if not dest_path: + continue + + # Keep plugin disabled/enabled status + match = re.match(re.escape(source_path) + "/plugins/([^/]+)", dest_path) + if match: + plugin_name = match.group(1).replace("disabled-", "") + if plugin_name in plugins_enabled: # Plugin was enabled + dest_path = dest_path.replace("plugins/disabled-" + plugin_name, "plugins/" + plugin_name) + elif plugin_name in plugins_disabled: # Plugin was disabled + dest_path = dest_path.replace("plugins/" + plugin_name, "plugins/disabled-" + plugin_name) + print("P", end=" ") + + dest_dir = os.path.dirname(dest_path) + if dest_dir and not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + + if dest_dir != dest_path.strip("/"): + update_paths[updatesite_path + "/" + inner_path] = dest_path + + num_ok = 0 + num_rename = 0 + num_error = 0 + for path_from, path_to in update_paths.items(): + print("-", path_from, "->", path_to) + if not os.path.isfile(path_from): + print("Missing file") + continue + + data = open(path_from, "rb").read() + + try: + open(path_to, 'wb').write(data) + num_ok += 1 + except Exception as err: + try: + print("Error writing: %s. Renaming old file as workaround..." % err) + path_to_tmp = path_to + "-old" + if os.path.isfile(path_to_tmp): + os.unlink(path_to_tmp) + os.rename(path_to, path_to_tmp) + num_rename += 1 + open(path_to, 'wb').write(data) + shutil.copymode(path_to_tmp, path_to) # Copy permissions + print("Write done after rename!") + num_ok += 1 + except Exception as err: + print("Write error after rename: %s" % err) + num_error += 1 + print("* Updated files: %s, renamed: %s, error: %s" % (num_ok, num_rename, num_error)) + + +if __name__ == "__main__": + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) # Imports relative to src + + update() diff --git a/zeronet.py b/zeronet.py new file mode 100755 index 00000000..457efb19 --- /dev/null +++ b/zeronet.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +import os +import sys + + +def main(): + if sys.version_info.major < 3: + print("Error: Python 3.x is required") + sys.exit(0) + + if "--silent" not in sys.argv: + print("- Starting ZeroNet...") + + main = None + try: + import main + main.start() + except Exception as err: # Prevent closing + import traceback + try: + import logging + logging.exception("Unhandled exception: %s" % err) + except Exception as log_err: + print("Failed to log error:", log_err) + traceback.print_exc() + from Config import config + error_log_path = config.log_dir + "/error.log" + traceback.print_exc(file=open(error_log_path, "w")) + print("---") + print("Please report it: https://github.com/HelloZeroNet/ZeroNet/issues/new?assignees=&labels=&template=bug-report.md") + if sys.platform.startswith("win") and "python.exe" not in sys.executable: + displayErrorMessage(err, error_log_path) + + if main and (main.update_after_shutdown or main.restart_after_shutdown): # Updater + if main.update_after_shutdown: + print("Shutting down...") + prepareShutdown() + import update + print("Updating...") + update.update() + if main.restart_after_shutdown: + print("Restarting...") + restart() + else: + print("Shutting down...") + prepareShutdown() + print("Restarting...") + restart() + + +def displayErrorMessage(err, error_log_path): + import ctypes + import urllib.parse + import subprocess + + MB_YESNOCANCEL = 0x3 + MB_ICONEXCLAIMATION = 0x30 + + ID_YES = 0x6 + ID_NO = 0x7 + ID_CANCEL = 0x2 + + err_message = "%s: %s" % (type(err).__name__, err) + err_title = "Unhandled exception: %s\nReport error?" % err_message + + res = ctypes.windll.user32.MessageBoxW(0, err_title, "ZeroNet error", MB_YESNOCANCEL | MB_ICONEXCLAIMATION) + if res == ID_YES: + import webbrowser + report_url = "https://github.com/ZeroNetX/ZeroNet/issues/new?assignees=&labels=&template=bug-report.md&title=%s" + webbrowser.open(report_url % urllib.parse.quote("Unhandled exception: %s" % err_message)) + if res in [ID_YES, ID_NO]: + subprocess.Popen(['notepad.exe', error_log_path]) + +def prepareShutdown(): + import atexit + atexit._run_exitfuncs() + + # Close log files + if "main" in sys.modules: + logger = sys.modules["main"].logging.getLogger() + + for handler in logger.handlers[:]: + handler.flush() + handler.close() + logger.removeHandler(handler) + + import time + time.sleep(1) # Wait for files to close + +def restart(): + args = sys.argv[:] + + sys.executable = sys.executable.replace(".pkg", "") # Frozen mac fix + + if not getattr(sys, 'frozen', False): + args.insert(0, sys.executable) + + # Don't open browser after restart + if "--open_browser" in args: + del args[args.index("--open_browser") + 1] # argument value + del args[args.index("--open_browser")] # argument key + + if getattr(sys, 'frozen', False): + pos_first_arg = 1 # Only the executable + else: + pos_first_arg = 2 # Interpter, .py file path + + args.insert(pos_first_arg, "--open_browser") + args.insert(pos_first_arg + 1, "False") + + if sys.platform == 'win32': + args = ['"%s"' % arg for arg in args] + + try: + print("Executing %s %s" % (sys.executable, args)) + os.execv(sys.executable, args) + except Exception as err: + print("Execv error: %s" % err) + print("Bye.") + + +def start(): + app_dir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(app_dir) # Change working dir to zeronet.py dir + sys.path.insert(0, os.path.join(app_dir, "src/lib")) # External liblary directory + sys.path.insert(0, os.path.join(app_dir, "src")) # Imports relative to src + + if "--update" in sys.argv: + sys.argv.remove("--update") + print("Updating...") + import update + update.update() + else: + main() + + +if __name__ == '__main__': + start()