diff --git a/plugins/Sidebar/maxminddb/__init__.py b/plugins/Sidebar/maxminddb/__init__.py deleted file mode 100644 index fc28186b..00000000 --- a/plugins/Sidebar/maxminddb/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# pylint:disable=C0111 -import os - -import maxminddb.reader - -try: - import maxminddb.extension -except ImportError: - maxminddb.extension = None - -from maxminddb.const import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE, - MODE_MEMORY) -from maxminddb.decoder import InvalidDatabaseError - - -def open_database(database, mode=MODE_AUTO): - """Open a Maxmind DB database - - Arguments: - database -- A path to a valid MaxMind DB file such as a GeoIP2 - database file. - mode -- mode to open the database with. Valid mode are: - * MODE_MMAP_EXT - use the C extension with memory map. - * MODE_MMAP - read from memory map. Pure Python. - * MODE_FILE - read database as standard file. Pure Python. - * MODE_MEMORY - load database into memory. Pure Python. - * MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that - order. Default mode. - """ - if (mode == MODE_AUTO and maxminddb.extension and - hasattr(maxminddb.extension, 'Reader')) or mode == MODE_MMAP_EXT: - return maxminddb.extension.Reader(database) - elif mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY): - return maxminddb.reader.Reader(database, mode) - raise ValueError('Unsupported open mode: {0}'.format(mode)) - - -def Reader(database): # pylint: disable=invalid-name - """This exists for backwards compatibility. Use open_database instead""" - return open_database(database) - -__title__ = 'maxminddb' -__version__ = '1.2.0' -__author__ = 'Gregory Oschwald' -__license__ = 'Apache License, Version 2.0' -__copyright__ = 'Copyright 2014 Maxmind, Inc.' diff --git a/plugins/Sidebar/maxminddb/compat.py b/plugins/Sidebar/maxminddb/compat.py deleted file mode 100644 index 14c98832..00000000 --- a/plugins/Sidebar/maxminddb/compat.py +++ /dev/null @@ -1,28 +0,0 @@ -import sys - -# pylint: skip-file - -if sys.version_info[0] == 2: - import ipaddr as ipaddress # pylint:disable=F0401 - ipaddress.ip_address = ipaddress.IPAddress - - int_from_byte = ord - - FileNotFoundError = IOError - - def int_from_bytes(b): - if b: - return int(b.encode("hex"), 16) - return 0 - - byte_from_int = chr -else: - import ipaddress # pylint:disable=F0401 - - int_from_byte = lambda x: x - - FileNotFoundError = FileNotFoundError - - int_from_bytes = lambda x: int.from_bytes(x, 'big') - - byte_from_int = lambda x: bytes([x]) diff --git a/plugins/Sidebar/maxminddb/const.py b/plugins/Sidebar/maxminddb/const.py deleted file mode 100644 index 59ea84b6..00000000 --- a/plugins/Sidebar/maxminddb/const.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Constants used in the API""" - -MODE_AUTO = 0 -MODE_MMAP_EXT = 1 -MODE_MMAP = 2 -MODE_FILE = 4 -MODE_MEMORY = 8 diff --git a/plugins/Sidebar/maxminddb/decoder.py b/plugins/Sidebar/maxminddb/decoder.py deleted file mode 100644 index 1b8f0711..00000000 --- a/plugins/Sidebar/maxminddb/decoder.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -maxminddb.decoder -~~~~~~~~~~~~~~~~~ - -This package contains code for decoding the MaxMind DB data section. - -""" -from __future__ import unicode_literals - -import struct - -from maxminddb.compat import byte_from_int, int_from_bytes -from maxminddb.errors import InvalidDatabaseError - - -class Decoder(object): # pylint: disable=too-few-public-methods - - """Decoder for the data section of the MaxMind DB""" - - def __init__(self, database_buffer, pointer_base=0, pointer_test=False): - """Created a Decoder for a MaxMind DB - - Arguments: - database_buffer -- an mmap'd MaxMind DB file. - pointer_base -- the base number to use when decoding a pointer - pointer_test -- used for internal unit testing of pointer code - """ - self._pointer_test = pointer_test - self._buffer = database_buffer - self._pointer_base = pointer_base - - def _decode_array(self, size, offset): - array = [] - for _ in range(size): - (value, offset) = self.decode(offset) - array.append(value) - return array, offset - - def _decode_boolean(self, size, offset): - return size != 0, offset - - def _decode_bytes(self, size, offset): - new_offset = offset + size - return self._buffer[offset:new_offset], new_offset - - # pylint: disable=no-self-argument - # |-> I am open to better ways of doing this as long as it doesn't involve - # lots of code duplication. - def _decode_packed_type(type_code, type_size, pad=False): - # pylint: disable=protected-access, missing-docstring - def unpack_type(self, size, offset): - if not pad: - self._verify_size(size, type_size) - new_offset = offset + type_size - packed_bytes = self._buffer[offset:new_offset] - if pad: - packed_bytes = packed_bytes.rjust(type_size, b'\x00') - (value,) = struct.unpack(type_code, packed_bytes) - return value, new_offset - return unpack_type - - def _decode_map(self, size, offset): - container = {} - for _ in range(size): - (key, offset) = self.decode(offset) - (value, offset) = self.decode(offset) - container[key] = value - return container, offset - - _pointer_value_offset = { - 1: 0, - 2: 2048, - 3: 526336, - 4: 0, - } - - def _decode_pointer(self, size, offset): - pointer_size = ((size >> 3) & 0x3) + 1 - new_offset = offset + pointer_size - pointer_bytes = self._buffer[offset:new_offset] - packed = pointer_bytes if pointer_size == 4 else struct.pack( - b'!c', byte_from_int(size & 0x7)) + pointer_bytes - unpacked = int_from_bytes(packed) - pointer = unpacked + self._pointer_base + \ - self._pointer_value_offset[pointer_size] - if self._pointer_test: - return pointer, new_offset - (value, _) = self.decode(pointer) - return value, new_offset - - def _decode_uint(self, size, offset): - new_offset = offset + size - uint_bytes = self._buffer[offset:new_offset] - return int_from_bytes(uint_bytes), new_offset - - def _decode_utf8_string(self, size, offset): - new_offset = offset + size - return self._buffer[offset:new_offset].decode('utf-8'), new_offset - - _type_decoder = { - 1: _decode_pointer, - 2: _decode_utf8_string, - 3: _decode_packed_type(b'!d', 8), # double, - 4: _decode_bytes, - 5: _decode_uint, # uint16 - 6: _decode_uint, # uint32 - 7: _decode_map, - 8: _decode_packed_type(b'!i', 4, pad=True), # int32 - 9: _decode_uint, # uint64 - 10: _decode_uint, # uint128 - 11: _decode_array, - 14: _decode_boolean, - 15: _decode_packed_type(b'!f', 4), # float, - } - - def decode(self, offset): - """Decode a section of the data section starting at offset - - Arguments: - offset -- the location of the data structure to decode - """ - new_offset = offset + 1 - (ctrl_byte,) = struct.unpack(b'!B', self._buffer[offset:new_offset]) - type_num = ctrl_byte >> 5 - # Extended type - if not type_num: - (type_num, new_offset) = self._read_extended(new_offset) - - if not type_num in self._type_decoder: - raise InvalidDatabaseError('Unexpected type number ({type}) ' - 'encountered'.format(type=type_num)) - - (size, new_offset) = self._size_from_ctrl_byte( - ctrl_byte, new_offset, type_num) - return self._type_decoder[type_num](self, size, new_offset) - - def _read_extended(self, offset): - (next_byte,) = struct.unpack(b'!B', self._buffer[offset:offset + 1]) - type_num = next_byte + 7 - if type_num < 7: - raise InvalidDatabaseError( - 'Something went horribly wrong in the decoder. An ' - 'extended type resolved to a type number < 8 ' - '({type})'.format(type=type_num)) - return type_num, offset + 1 - - def _verify_size(self, expected, actual): - if expected != actual: - raise InvalidDatabaseError( - 'The MaxMind DB file\'s data section contains bad data ' - '(unknown data type or corrupt data)' - ) - - def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num): - size = ctrl_byte & 0x1f - if type_num == 1: - return size, offset - bytes_to_read = 0 if size < 29 else size - 28 - - new_offset = offset + bytes_to_read - size_bytes = self._buffer[offset:new_offset] - - # Using unpack rather than int_from_bytes as it is about 200 lookups - # per second faster here. - if size == 29: - size = 29 + struct.unpack(b'!B', size_bytes)[0] - elif size == 30: - size = 285 + struct.unpack(b'!H', size_bytes)[0] - elif size > 30: - size = struct.unpack( - b'!I', size_bytes.rjust(4, b'\x00'))[0] + 65821 - - return size, new_offset diff --git a/plugins/Sidebar/maxminddb/errors.py b/plugins/Sidebar/maxminddb/errors.py deleted file mode 100644 index f04ff028..00000000 --- a/plugins/Sidebar/maxminddb/errors.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -maxminddb.errors -~~~~~~~~~~~~~~~~ - -This module contains custom errors for the MaxMind DB reader -""" - - -class InvalidDatabaseError(RuntimeError): - - """This error is thrown when unexpected data is found in the database.""" diff --git a/plugins/Sidebar/maxminddb/extension/maxminddb.c b/plugins/Sidebar/maxminddb/extension/maxminddb.c deleted file mode 100644 index 9e4d45e2..00000000 --- a/plugins/Sidebar/maxminddb/extension/maxminddb.c +++ /dev/null @@ -1,570 +0,0 @@ -#include -#include -#include "structmember.h" - -#define __STDC_FORMAT_MACROS -#include - -static PyTypeObject Reader_Type; -static PyTypeObject Metadata_Type; -static PyObject *MaxMindDB_error; - -typedef struct { - PyObject_HEAD /* no semicolon */ - MMDB_s *mmdb; -} Reader_obj; - -typedef struct { - PyObject_HEAD /* no semicolon */ - PyObject *binary_format_major_version; - PyObject *binary_format_minor_version; - PyObject *build_epoch; - PyObject *database_type; - PyObject *description; - PyObject *ip_version; - PyObject *languages; - PyObject *node_count; - PyObject *record_size; -} Metadata_obj; - -static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list); - -#if PY_MAJOR_VERSION >= 3 - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_ ## name(void) - #define RETURN_MOD_INIT(m) return (m) - #define FILE_NOT_FOUND_ERROR PyExc_FileNotFoundError -#else - #define MOD_INIT(name) PyMODINIT_FUNC init ## name(void) - #define RETURN_MOD_INIT(m) return - #define PyInt_FromLong PyLong_FromLong - #define FILE_NOT_FOUND_ERROR PyExc_IOError -#endif - -#ifdef __GNUC__ - # define UNUSED(x) UNUSED_ ## x __attribute__((__unused__)) -#else - # define UNUSED(x) UNUSED_ ## x -#endif - -static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - char *filename; - int mode = 0; - - static char *kwlist[] = {"database", "mode", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", kwlist, &filename, &mode)) { - return -1; - } - - if (mode != 0 && mode != 1) { - PyErr_Format(PyExc_ValueError, "Unsupported open mode (%i). Only " - "MODE_AUTO and MODE_MMAP_EXT are supported by this extension.", - mode); - return -1; - } - - if (0 != access(filename, R_OK)) { - PyErr_Format(FILE_NOT_FOUND_ERROR, - "No such file or directory: '%s'", - filename); - return -1; - } - - MMDB_s *mmdb = (MMDB_s *)malloc(sizeof(MMDB_s)); - if (NULL == mmdb) { - PyErr_NoMemory(); - return -1; - } - - Reader_obj *mmdb_obj = (Reader_obj *)self; - if (!mmdb_obj) { - free(mmdb); - PyErr_NoMemory(); - return -1; - } - - uint16_t status = MMDB_open(filename, MMDB_MODE_MMAP, mmdb); - - if (MMDB_SUCCESS != status) { - free(mmdb); - PyErr_Format( - MaxMindDB_error, - "Error opening database file (%s). Is this a valid MaxMind DB file?", - filename - ); - return -1; - } - - mmdb_obj->mmdb = mmdb; - return 0; -} - -static PyObject *Reader_get(PyObject *self, PyObject *args) -{ - char *ip_address = NULL; - - Reader_obj *mmdb_obj = (Reader_obj *)self; - if (!PyArg_ParseTuple(args, "s", &ip_address)) { - return NULL; - } - - MMDB_s *mmdb = mmdb_obj->mmdb; - - if (NULL == mmdb) { - PyErr_SetString(PyExc_ValueError, - "Attempt to read from a closed MaxMind DB."); - return NULL; - } - - int gai_error = 0; - int mmdb_error = MMDB_SUCCESS; - MMDB_lookup_result_s result = - MMDB_lookup_string(mmdb, ip_address, &gai_error, - &mmdb_error); - - if (0 != gai_error) { - PyErr_Format(PyExc_ValueError, - "'%s' does not appear to be an IPv4 or IPv6 address.", - ip_address); - return NULL; - } - - if (MMDB_SUCCESS != mmdb_error) { - PyObject *exception; - if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) { - exception = PyExc_ValueError; - } else { - exception = MaxMindDB_error; - } - PyErr_Format(exception, "Error looking up %s. %s", - ip_address, MMDB_strerror(mmdb_error)); - return NULL; - } - - if (!result.found_entry) { - Py_RETURN_NONE; - } - - MMDB_entry_data_list_s *entry_data_list = NULL; - int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); - if (MMDB_SUCCESS != status) { - PyErr_Format(MaxMindDB_error, - "Error while looking up data for %s. %s", - ip_address, MMDB_strerror(status)); - MMDB_free_entry_data_list(entry_data_list); - return NULL; - } - - MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - PyObject *py_obj = from_entry_data_list(&entry_data_list); - MMDB_free_entry_data_list(original_entry_data_list); - return py_obj; -} - -static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) -{ - Reader_obj *mmdb_obj = (Reader_obj *)self; - - if (NULL == mmdb_obj->mmdb) { - PyErr_SetString(PyExc_IOError, - "Attempt to read from a closed MaxMind DB."); - return NULL; - } - - MMDB_entry_data_list_s *entry_data_list; - MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list); - MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - - PyObject *metadata_dict = from_entry_data_list(&entry_data_list); - MMDB_free_entry_data_list(original_entry_data_list); - if (NULL == metadata_dict || !PyDict_Check(metadata_dict)) { - PyErr_SetString(MaxMindDB_error, - "Error decoding metadata."); - return NULL; - } - - PyObject *args = PyTuple_New(0); - if (NULL == args) { - Py_DECREF(metadata_dict); - return NULL; - } - - PyObject *metadata = PyObject_Call((PyObject *)&Metadata_Type, args, - metadata_dict); - - Py_DECREF(metadata_dict); - return metadata; -} - -static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) -{ - Reader_obj *mmdb_obj = (Reader_obj *)self; - - if (NULL != mmdb_obj->mmdb) { - MMDB_close(mmdb_obj->mmdb); - free(mmdb_obj->mmdb); - mmdb_obj->mmdb = NULL; - } - - Py_RETURN_NONE; -} - -static void Reader_dealloc(PyObject *self) -{ - Reader_obj *obj = (Reader_obj *)self; - if (NULL != obj->mmdb) { - Reader_close(self, NULL); - } - - PyObject_Del(self); -} - -static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - - PyObject - *binary_format_major_version, - *binary_format_minor_version, - *build_epoch, - *database_type, - *description, - *ip_version, - *languages, - *node_count, - *record_size; - - static char *kwlist[] = { - "binary_format_major_version", - "binary_format_minor_version", - "build_epoch", - "database_type", - "description", - "ip_version", - "languages", - "node_count", - "record_size", - NULL - }; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOOOOOO", kwlist, - &binary_format_major_version, - &binary_format_minor_version, - &build_epoch, - &database_type, - &description, - &ip_version, - &languages, - &node_count, - &record_size)) { - return -1; - } - - Metadata_obj *obj = (Metadata_obj *)self; - - obj->binary_format_major_version = binary_format_major_version; - obj->binary_format_minor_version = binary_format_minor_version; - obj->build_epoch = build_epoch; - obj->database_type = database_type; - obj->description = description; - obj->ip_version = ip_version; - obj->languages = languages; - obj->node_count = node_count; - obj->record_size = record_size; - - Py_INCREF(obj->binary_format_major_version); - Py_INCREF(obj->binary_format_minor_version); - Py_INCREF(obj->build_epoch); - Py_INCREF(obj->database_type); - Py_INCREF(obj->description); - Py_INCREF(obj->ip_version); - Py_INCREF(obj->languages); - Py_INCREF(obj->node_count); - Py_INCREF(obj->record_size); - - return 0; -} - -static void Metadata_dealloc(PyObject *self) -{ - Metadata_obj *obj = (Metadata_obj *)self; - Py_DECREF(obj->binary_format_major_version); - Py_DECREF(obj->binary_format_minor_version); - Py_DECREF(obj->build_epoch); - Py_DECREF(obj->database_type); - Py_DECREF(obj->description); - Py_DECREF(obj->ip_version); - Py_DECREF(obj->languages); - Py_DECREF(obj->node_count); - Py_DECREF(obj->record_size); - PyObject_Del(self); -} - -static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) -{ - if (NULL == entry_data_list || NULL == *entry_data_list) { - PyErr_SetString( - MaxMindDB_error, - "Error while looking up data. Your database may be corrupt or you have found a bug in libmaxminddb." - ); - return NULL; - } - - switch ((*entry_data_list)->entry_data.type) { - case MMDB_DATA_TYPE_MAP: - return from_map(entry_data_list); - case MMDB_DATA_TYPE_ARRAY: - return from_array(entry_data_list); - case MMDB_DATA_TYPE_UTF8_STRING: - return PyUnicode_FromStringAndSize( - (*entry_data_list)->entry_data.utf8_string, - (*entry_data_list)->entry_data.data_size - ); - case MMDB_DATA_TYPE_BYTES: - return PyByteArray_FromStringAndSize( - (const char *)(*entry_data_list)->entry_data.bytes, - (Py_ssize_t)(*entry_data_list)->entry_data.data_size); - case MMDB_DATA_TYPE_DOUBLE: - return PyFloat_FromDouble((*entry_data_list)->entry_data.double_value); - case MMDB_DATA_TYPE_FLOAT: - return PyFloat_FromDouble((*entry_data_list)->entry_data.float_value); - case MMDB_DATA_TYPE_UINT16: - return PyLong_FromLong( (*entry_data_list)->entry_data.uint16); - case MMDB_DATA_TYPE_UINT32: - return PyLong_FromLong((*entry_data_list)->entry_data.uint32); - case MMDB_DATA_TYPE_BOOLEAN: - return PyBool_FromLong((*entry_data_list)->entry_data.boolean); - case MMDB_DATA_TYPE_UINT64: - return PyLong_FromUnsignedLongLong( - (*entry_data_list)->entry_data.uint64); - case MMDB_DATA_TYPE_UINT128: - return from_uint128(*entry_data_list); - case MMDB_DATA_TYPE_INT32: - return PyLong_FromLong((*entry_data_list)->entry_data.int32); - default: - PyErr_Format(MaxMindDB_error, - "Invalid data type arguments: %d", - (*entry_data_list)->entry_data.type); - return NULL; - } - return NULL; -} - -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) -{ - PyObject *py_obj = PyDict_New(); - if (NULL == py_obj) { - PyErr_NoMemory(); - return NULL; - } - - const uint32_t map_size = (*entry_data_list)->entry_data.data_size; - - uint i; - // entry_data_list cannot start out NULL (see from_entry_data_list). We - // check it in the loop because it may become NULL. - // coverity[check_after_deref] - for (i = 0; i < map_size && entry_data_list; i++) { - *entry_data_list = (*entry_data_list)->next; - - PyObject *key = PyUnicode_FromStringAndSize( - (char *)(*entry_data_list)->entry_data.utf8_string, - (*entry_data_list)->entry_data.data_size - ); - - *entry_data_list = (*entry_data_list)->next; - - PyObject *value = from_entry_data_list(entry_data_list); - if (NULL == value) { - Py_DECREF(key); - Py_DECREF(py_obj); - return NULL; - } - PyDict_SetItem(py_obj, key, value); - Py_DECREF(value); - Py_DECREF(key); - } - - return py_obj; -} - -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) -{ - const uint32_t size = (*entry_data_list)->entry_data.data_size; - - PyObject *py_obj = PyList_New(size); - if (NULL == py_obj) { - PyErr_NoMemory(); - return NULL; - } - - uint i; - // entry_data_list cannot start out NULL (see from_entry_data_list). We - // check it in the loop because it may become NULL. - // coverity[check_after_deref] - for (i = 0; i < size && entry_data_list; i++) { - *entry_data_list = (*entry_data_list)->next; - PyObject *value = from_entry_data_list(entry_data_list); - if (NULL == value) { - Py_DECREF(py_obj); - return NULL; - } - // PyList_SetItem 'steals' the reference - PyList_SetItem(py_obj, i, value); - } - return py_obj; -} - -static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) -{ - uint64_t high = 0; - uint64_t low = 0; -#if MMDB_UINT128_IS_BYTE_ARRAY - int i; - for (i = 0; i < 8; i++) { - high = (high << 8) | entry_data_list->entry_data.uint128[i]; - } - - for (i = 8; i < 16; i++) { - low = (low << 8) | entry_data_list->entry_data.uint128[i]; - } -#else - high = entry_data_list->entry_data.uint128 >> 64; - low = (uint64_t)entry_data_list->entry_data.uint128; -#endif - - char *num_str = malloc(33); - if (NULL == num_str) { - PyErr_NoMemory(); - return NULL; - } - - snprintf(num_str, 33, "%016" PRIX64 "%016" PRIX64, high, low); - - PyObject *py_obj = PyLong_FromString(num_str, NULL, 16); - - free(num_str); - return py_obj; -} - -static PyMethodDef Reader_methods[] = { - { "get", Reader_get, METH_VARARGS, - "Get record for IP address" }, - { "metadata", Reader_metadata, METH_NOARGS, - "Returns metadata object for database" }, - { "close", Reader_close, METH_NOARGS, "Closes database"}, - { NULL, NULL, 0, NULL } -}; - -static PyTypeObject Reader_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(Reader_obj), - .tp_dealloc = Reader_dealloc, - .tp_doc = "Reader object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_methods = Reader_methods, - .tp_name = "Reader", - .tp_init = Reader_init, -}; - -static PyMethodDef Metadata_methods[] = { - { NULL, NULL, 0, NULL } -}; - -/* *INDENT-OFF* */ -static PyMemberDef Metadata_members[] = { - { "binary_format_major_version", T_OBJECT, offsetof( - Metadata_obj, binary_format_major_version), READONLY, NULL }, - { "binary_format_minor_version", T_OBJECT, offsetof( - Metadata_obj, binary_format_minor_version), READONLY, NULL }, - { "build_epoch", T_OBJECT, offsetof(Metadata_obj, build_epoch), - READONLY, NULL }, - { "database_type", T_OBJECT, offsetof(Metadata_obj, database_type), - READONLY, NULL }, - { "description", T_OBJECT, offsetof(Metadata_obj, description), - READONLY, NULL }, - { "ip_version", T_OBJECT, offsetof(Metadata_obj, ip_version), - READONLY, NULL }, - { "languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY, - NULL }, - { "node_count", T_OBJECT, offsetof(Metadata_obj, node_count), - READONLY, NULL }, - { "record_size", T_OBJECT, offsetof(Metadata_obj, record_size), - READONLY, NULL }, - { NULL, 0, 0, 0, NULL } -}; -/* *INDENT-ON* */ - -static PyTypeObject Metadata_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(Metadata_obj), - .tp_dealloc = Metadata_dealloc, - .tp_doc = "Metadata object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_members = Metadata_members, - .tp_methods = Metadata_methods, - .tp_name = "Metadata", - .tp_init = Metadata_init -}; - -static PyMethodDef MaxMindDB_methods[] = { - { NULL, NULL, 0, NULL } -}; - - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef MaxMindDB_module = { - PyModuleDef_HEAD_INIT, - .m_name = "extension", - .m_doc = "This is a C extension to read MaxMind DB file format", - .m_methods = MaxMindDB_methods, -}; -#endif - -MOD_INIT(extension){ - PyObject *m; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&MaxMindDB_module); -#else - m = Py_InitModule("extension", MaxMindDB_methods); -#endif - - if (!m) { - RETURN_MOD_INIT(NULL); - } - - Reader_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&Reader_Type)) { - RETURN_MOD_INIT(NULL); - } - Py_INCREF(&Reader_Type); - PyModule_AddObject(m, "Reader", (PyObject *)&Reader_Type); - - Metadata_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&Metadata_Type)) { - RETURN_MOD_INIT(NULL); - } - PyModule_AddObject(m, "extension", (PyObject *)&Metadata_Type); - - PyObject* error_mod = PyImport_ImportModule("maxminddb.errors"); - if (error_mod == NULL) { - RETURN_MOD_INIT(NULL); - } - - MaxMindDB_error = PyObject_GetAttrString(error_mod, "InvalidDatabaseError"); - Py_DECREF(error_mod); - - if (MaxMindDB_error == NULL) { - RETURN_MOD_INIT(NULL); - } - - Py_INCREF(MaxMindDB_error); - - /* We primarily add it to the module for backwards compatibility */ - PyModule_AddObject(m, "InvalidDatabaseError", MaxMindDB_error); - - RETURN_MOD_INIT(m); -} diff --git a/plugins/Sidebar/maxminddb/file.py b/plugins/Sidebar/maxminddb/file.py deleted file mode 100644 index 3460893e..00000000 --- a/plugins/Sidebar/maxminddb/file.py +++ /dev/null @@ -1,65 +0,0 @@ -"""For internal use only. It provides a slice-like file reader.""" - -import os - -try: - from multiprocessing import Lock -except ImportError: - from threading import Lock - - -class FileBuffer(object): - - """A slice-able file reader""" - - def __init__(self, database): - self._handle = open(database, 'rb') - self._size = os.fstat(self._handle.fileno()).st_size - if not hasattr(os, 'pread'): - self._lock = Lock() - - def __getitem__(self, key): - if isinstance(key, slice): - return self._read(key.stop - key.start, key.start) - elif isinstance(key, int): - return self._read(1, key) - else: - raise TypeError("Invalid argument type.") - - def rfind(self, needle, start): - """Reverse find needle from start""" - pos = self._read(self._size - start - 1, start).rfind(needle) - if pos == -1: - return pos - return start + pos - - def size(self): - """Size of file""" - return self._size - - def close(self): - """Close file""" - self._handle.close() - - if hasattr(os, 'pread'): - - def _read(self, buffersize, offset): - """read that uses pread""" - # pylint: disable=no-member - return os.pread(self._handle.fileno(), buffersize, offset) - - else: - - def _read(self, buffersize, offset): - """read with a lock - - This lock is necessary as after a fork, the different processes - will share the same file table entry, even if we dup the fd, and - as such the same offsets. There does not appear to be a way to - duplicate the file table entry and we cannot re-open based on the - original path as that file may have replaced with another or - unlinked. - """ - with self._lock: - self._handle.seek(offset) - return self._handle.read(buffersize) diff --git a/plugins/Sidebar/maxminddb/ipaddr.py b/plugins/Sidebar/maxminddb/ipaddr.py deleted file mode 100644 index ad27ae9d..00000000 --- a/plugins/Sidebar/maxminddb/ipaddr.py +++ /dev/null @@ -1,1897 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2007 Google Inc. -# Licensed to PSF under a Contributor Agreement. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. - -"""A fast, lightweight IPv4/IPv6 manipulation library in Python. - -This library is used to create/poke/manipulate IPv4 and IPv6 addresses -and networks. - -""" - -__version__ = '2.1.10' - -import struct - -IPV4LENGTH = 32 -IPV6LENGTH = 128 - - -class AddressValueError(ValueError): - """A Value Error related to the address.""" - - -class NetmaskValueError(ValueError): - """A Value Error related to the netmask.""" - - -def IPAddress(address, version=None): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP address. Either IPv4 or - IPv6 addresses may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - version: An Integer, 4 or 6. If set, don't try to automatically - determine what the IP address type is. important for things - like IPAddress(1), which could be IPv4, '0.0.0.1', or IPv6, - '::1'. - - Returns: - An IPv4Address or IPv6Address object. - - Raises: - ValueError: if the string passed isn't either a v4 or a v6 - address. - - """ - if version: - if version == 4: - return IPv4Address(address) - elif version == 6: - return IPv6Address(address) - - try: - return IPv4Address(address) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Address(address) - except (AddressValueError, NetmaskValueError): - pass - - raise ValueError('%r does not appear to be an IPv4 or IPv6 address' % - address) - - -def IPNetwork(address, version=None, strict=False): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP address. Either IPv4 or - IPv6 addresses may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - version: An Integer, if set, don't try to automatically - determine what the IP address type is. important for things - like IPNetwork(1), which could be IPv4, '0.0.0.1/32', or IPv6, - '::1/128'. - - Returns: - An IPv4Network or IPv6Network object. - - Raises: - ValueError: if the string passed isn't either a v4 or a v6 - address. Or if a strict network was requested and a strict - network wasn't given. - - """ - if version: - if version == 4: - return IPv4Network(address, strict) - elif version == 6: - return IPv6Network(address, strict) - - try: - return IPv4Network(address, strict) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Network(address, strict) - except (AddressValueError, NetmaskValueError): - pass - - raise ValueError('%r does not appear to be an IPv4 or IPv6 network' % - address) - - -def v4_int_to_packed(address): - """The binary representation of this address. - - Args: - address: An integer representation of an IPv4 IP address. - - Returns: - The binary representation of this address. - - Raises: - ValueError: If the integer is too large to be an IPv4 IP - address. - """ - if address > _BaseV4._ALL_ONES: - raise ValueError('Address too large for IPv4') - return Bytes(struct.pack('!I', address)) - - -def v6_int_to_packed(address): - """The binary representation of this address. - - Args: - address: An integer representation of an IPv4 IP address. - - Returns: - The binary representation of this address. - """ - return Bytes(struct.pack('!QQ', address >> 64, address & (2**64 - 1))) - - -def _find_address_range(addresses): - """Find a sequence of addresses. - - Args: - addresses: a list of IPv4 or IPv6 addresses. - - Returns: - A tuple containing the first and last IP addresses in the sequence. - - """ - first = last = addresses[0] - for ip in addresses[1:]: - if ip._ip == last._ip + 1: - last = ip - else: - break - return (first, last) - -def _get_prefix_length(number1, number2, bits): - """Get the number of leading bits that are same for two numbers. - - Args: - number1: an integer. - number2: another integer. - bits: the maximum number of bits to compare. - - Returns: - The number of leading bits that are the same for two numbers. - - """ - for i in range(bits): - if number1 >> i == number2 >> i: - return bits - i - return 0 - -def _count_righthand_zero_bits(number, bits): - """Count the number of zero bits on the right hand side. - - Args: - number: an integer. - bits: maximum number of bits to count. - - Returns: - The number of zero bits on the right hand side of the number. - - """ - if number == 0: - return bits - for i in range(bits): - if (number >> i) % 2: - return i - -def summarize_address_range(first, last): - """Summarize a network range given the first and last IP addresses. - - Example: - >>> summarize_address_range(IPv4Address('1.1.1.0'), - IPv4Address('1.1.1.130')) - [IPv4Network('1.1.1.0/25'), IPv4Network('1.1.1.128/31'), - IPv4Network('1.1.1.130/32')] - - Args: - first: the first IPv4Address or IPv6Address in the range. - last: the last IPv4Address or IPv6Address in the range. - - Returns: - The address range collapsed to a list of IPv4Network's or - IPv6Network's. - - Raise: - TypeError: - If the first and last objects are not IP addresses. - If the first and last objects are not the same version. - ValueError: - If the last object is not greater than the first. - If the version is not 4 or 6. - - """ - if not (isinstance(first, _BaseIP) and isinstance(last, _BaseIP)): - raise TypeError('first and last must be IP addresses, not networks') - if first.version != last.version: - raise TypeError("%s and %s are not of the same version" % ( - str(first), str(last))) - if first > last: - raise ValueError('last IP address must be greater than first') - - networks = [] - - if first.version == 4: - ip = IPv4Network - elif first.version == 6: - ip = IPv6Network - else: - raise ValueError('unknown IP version') - - ip_bits = first._max_prefixlen - first_int = first._ip - last_int = last._ip - while first_int <= last_int: - nbits = _count_righthand_zero_bits(first_int, ip_bits) - current = None - while nbits >= 0: - addend = 2**nbits - 1 - current = first_int + addend - nbits -= 1 - if current <= last_int: - break - prefix = _get_prefix_length(first_int, current, ip_bits) - net = ip('%s/%d' % (str(first), prefix)) - networks.append(net) - if current == ip._ALL_ONES: - break - first_int = current + 1 - first = IPAddress(first_int, version=first._version) - return networks - -def _collapse_address_list_recursive(addresses): - """Loops through the addresses, collapsing concurrent netblocks. - - Example: - - ip1 = IPv4Network('1.1.0.0/24') - ip2 = IPv4Network('1.1.1.0/24') - ip3 = IPv4Network('1.1.2.0/24') - ip4 = IPv4Network('1.1.3.0/24') - ip5 = IPv4Network('1.1.4.0/24') - ip6 = IPv4Network('1.1.0.1/22') - - _collapse_address_list_recursive([ip1, ip2, ip3, ip4, ip5, ip6]) -> - [IPv4Network('1.1.0.0/22'), IPv4Network('1.1.4.0/24')] - - This shouldn't be called directly; it is called via - collapse_address_list([]). - - Args: - addresses: A list of IPv4Network's or IPv6Network's - - Returns: - A list of IPv4Network's or IPv6Network's depending on what we were - passed. - - """ - ret_array = [] - optimized = False - - for cur_addr in addresses: - if not ret_array: - ret_array.append(cur_addr) - continue - if cur_addr in ret_array[-1]: - optimized = True - elif cur_addr == ret_array[-1].supernet().subnet()[1]: - ret_array.append(ret_array.pop().supernet()) - optimized = True - else: - ret_array.append(cur_addr) - - if optimized: - return _collapse_address_list_recursive(ret_array) - - return ret_array - - -def collapse_address_list(addresses): - """Collapse a list of IP objects. - - Example: - collapse_address_list([IPv4('1.1.0.0/24'), IPv4('1.1.1.0/24')]) -> - [IPv4('1.1.0.0/23')] - - Args: - addresses: A list of IPv4Network or IPv6Network objects. - - Returns: - A list of IPv4Network or IPv6Network objects depending on what we - were passed. - - Raises: - TypeError: If passed a list of mixed version objects. - - """ - i = 0 - addrs = [] - ips = [] - nets = [] - - # split IP addresses and networks - for ip in addresses: - if isinstance(ip, _BaseIP): - if ips and ips[-1]._version != ip._version: - raise TypeError("%s and %s are not of the same version" % ( - str(ip), str(ips[-1]))) - ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: - raise TypeError("%s and %s are not of the same version" % ( - str(ip), str(ips[-1]))) - ips.append(ip.ip) - else: - if nets and nets[-1]._version != ip._version: - raise TypeError("%s and %s are not of the same version" % ( - str(ip), str(ips[-1]))) - nets.append(ip) - - # sort and dedup - ips = sorted(set(ips)) - nets = sorted(set(nets)) - - while i < len(ips): - (first, last) = _find_address_range(ips[i:]) - i = ips.index(last) + 1 - addrs.extend(summarize_address_range(first, last)) - - return _collapse_address_list_recursive(sorted( - addrs + nets, key=_BaseNet._get_networks_key)) - -# backwards compatibility -CollapseAddrList = collapse_address_list - -# We need to distinguish between the string and packed-bytes representations -# of an IP address. For example, b'0::1' is the IPv4 address 48.58.58.49, -# while '0::1' is an IPv6 address. -# -# In Python 3, the native 'bytes' type already provides this functionality, -# so we use it directly. For earlier implementations where bytes is not a -# distinct type, we create a subclass of str to serve as a tag. -# -# Usage example (Python 2): -# ip = ipaddr.IPAddress(ipaddr.Bytes('xxxx')) -# -# Usage example (Python 3): -# ip = ipaddr.IPAddress(b'xxxx') -try: - if bytes is str: - raise TypeError("bytes is not a distinct type") - Bytes = bytes -except (NameError, TypeError): - class Bytes(str): - def __repr__(self): - return 'Bytes(%s)' % str.__repr__(self) - -def get_mixed_type_key(obj): - """Return a key suitable for sorting between networks and addresses. - - Address and Network objects are not sortable by default; they're - fundamentally different so the expression - - IPv4Address('1.1.1.1') <= IPv4Network('1.1.1.1/24') - - doesn't make any sense. There are some times however, where you may wish - to have ipaddr sort these for you anyway. If you need to do this, you - can use this function as the key= argument to sorted(). - - Args: - obj: either a Network or Address object. - Returns: - appropriate key. - - """ - if isinstance(obj, _BaseNet): - return obj._get_networks_key() - elif isinstance(obj, _BaseIP): - return obj._get_address_key() - return NotImplemented - -class _IPAddrBase(object): - - """The mother class.""" - - def __index__(self): - return self._ip - - def __int__(self): - return self._ip - - def __hex__(self): - return hex(self._ip) - - @property - def exploded(self): - """Return the longhand version of the IP address as a string.""" - return self._explode_shorthand_ip_string() - - @property - def compressed(self): - """Return the shorthand version of the IP address as a string.""" - return str(self) - - -class _BaseIP(_IPAddrBase): - - """A generic IP object. - - This IP class contains the version independent methods which are - used by single IP addresses. - - """ - - def __eq__(self, other): - try: - return (self._ip == other._ip - and self._version == other._version) - except AttributeError: - return NotImplemented - - def __ne__(self, other): - eq = self.__eq__(other) - if eq is NotImplemented: - return NotImplemented - return not eq - - def __le__(self, other): - gt = self.__gt__(other) - if gt is NotImplemented: - return NotImplemented - return not gt - - def __ge__(self, other): - lt = self.__lt__(other) - if lt is NotImplemented: - return NotImplemented - return not lt - - def __lt__(self, other): - if self._version != other._version: - raise TypeError('%s and %s are not of the same version' % ( - str(self), str(other))) - if not isinstance(other, _BaseIP): - raise TypeError('%s and %s are not of the same type' % ( - str(self), str(other))) - if self._ip != other._ip: - return self._ip < other._ip - return False - - def __gt__(self, other): - if self._version != other._version: - raise TypeError('%s and %s are not of the same version' % ( - str(self), str(other))) - if not isinstance(other, _BaseIP): - raise TypeError('%s and %s are not of the same type' % ( - str(self), str(other))) - if self._ip != other._ip: - return self._ip > other._ip - return False - - # Shorthand for Integer addition and subtraction. This is not - # meant to ever support addition/subtraction of addresses. - def __add__(self, other): - if not isinstance(other, int): - return NotImplemented - return IPAddress(int(self) + other, version=self._version) - - def __sub__(self, other): - if not isinstance(other, int): - return NotImplemented - return IPAddress(int(self) - other, version=self._version) - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, str(self)) - - def __str__(self): - return '%s' % self._string_from_ip_int(self._ip) - - def __hash__(self): - return hash(hex(long(self._ip))) - - def _get_address_key(self): - return (self._version, self) - - @property - def version(self): - raise NotImplementedError('BaseIP has no version') - - -class _BaseNet(_IPAddrBase): - - """A generic IP object. - - This IP class contains the version independent methods which are - used by networks. - - """ - - def __init__(self, address): - self._cache = {} - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, str(self)) - - def iterhosts(self): - """Generate Iterator over usable hosts in a network. - - This is like __iter__ except it doesn't return the network - or broadcast addresses. - - """ - cur = int(self.network) + 1 - bcast = int(self.broadcast) - 1 - while cur <= bcast: - cur += 1 - yield IPAddress(cur - 1, version=self._version) - - def __iter__(self): - cur = int(self.network) - bcast = int(self.broadcast) - while cur <= bcast: - cur += 1 - yield IPAddress(cur - 1, version=self._version) - - def __getitem__(self, n): - network = int(self.network) - broadcast = int(self.broadcast) - if n >= 0: - if network + n > broadcast: - raise IndexError - return IPAddress(network + n, version=self._version) - else: - n += 1 - if broadcast + n < network: - raise IndexError - return IPAddress(broadcast + n, version=self._version) - - def __lt__(self, other): - if self._version != other._version: - raise TypeError('%s and %s are not of the same version' % ( - str(self), str(other))) - if not isinstance(other, _BaseNet): - raise TypeError('%s and %s are not of the same type' % ( - str(self), str(other))) - if self.network != other.network: - return self.network < other.network - if self.netmask != other.netmask: - return self.netmask < other.netmask - return False - - def __gt__(self, other): - if self._version != other._version: - raise TypeError('%s and %s are not of the same version' % ( - str(self), str(other))) - if not isinstance(other, _BaseNet): - raise TypeError('%s and %s are not of the same type' % ( - str(self), str(other))) - if self.network != other.network: - return self.network > other.network - if self.netmask != other.netmask: - return self.netmask > other.netmask - return False - - def __le__(self, other): - gt = self.__gt__(other) - if gt is NotImplemented: - return NotImplemented - return not gt - - def __ge__(self, other): - lt = self.__lt__(other) - if lt is NotImplemented: - return NotImplemented - return not lt - - def __eq__(self, other): - try: - return (self._version == other._version - and self.network == other.network - and int(self.netmask) == int(other.netmask)) - except AttributeError: - if isinstance(other, _BaseIP): - return (self._version == other._version - and self._ip == other._ip) - - def __ne__(self, other): - eq = self.__eq__(other) - if eq is NotImplemented: - return NotImplemented - return not eq - - def __str__(self): - return '%s/%s' % (str(self.ip), - str(self._prefixlen)) - - def __hash__(self): - return hash(int(self.network) ^ int(self.netmask)) - - def __contains__(self, other): - # always false if one is v4 and the other is v6. - if self._version != other._version: - return False - # dealing with another network. - if isinstance(other, _BaseNet): - return (self.network <= other.network and - self.broadcast >= other.broadcast) - # dealing with another address - else: - return (int(self.network) <= int(other._ip) <= - int(self.broadcast)) - - def overlaps(self, other): - """Tell if self is partly contained in other.""" - return self.network in other or self.broadcast in other or ( - other.network in self or other.broadcast in self) - - @property - def network(self): - x = self._cache.get('network') - if x is None: - x = IPAddress(self._ip & int(self.netmask), version=self._version) - self._cache['network'] = x - return x - - @property - def broadcast(self): - x = self._cache.get('broadcast') - if x is None: - x = IPAddress(self._ip | int(self.hostmask), version=self._version) - self._cache['broadcast'] = x - return x - - @property - def hostmask(self): - x = self._cache.get('hostmask') - if x is None: - x = IPAddress(int(self.netmask) ^ self._ALL_ONES, - version=self._version) - self._cache['hostmask'] = x - return x - - @property - def with_prefixlen(self): - return '%s/%d' % (str(self.ip), self._prefixlen) - - @property - def with_netmask(self): - return '%s/%s' % (str(self.ip), str(self.netmask)) - - @property - def with_hostmask(self): - return '%s/%s' % (str(self.ip), str(self.hostmask)) - - @property - def numhosts(self): - """Number of hosts in the current subnet.""" - return int(self.broadcast) - int(self.network) + 1 - - @property - def version(self): - raise NotImplementedError('BaseNet has no version') - - @property - def prefixlen(self): - return self._prefixlen - - def address_exclude(self, other): - """Remove an address from a larger block. - - For example: - - addr1 = IPNetwork('10.1.1.0/24') - addr2 = IPNetwork('10.1.1.0/26') - addr1.address_exclude(addr2) = - [IPNetwork('10.1.1.64/26'), IPNetwork('10.1.1.128/25')] - - or IPv6: - - addr1 = IPNetwork('::1/32') - addr2 = IPNetwork('::1/128') - addr1.address_exclude(addr2) = [IPNetwork('::0/128'), - IPNetwork('::2/127'), - IPNetwork('::4/126'), - IPNetwork('::8/125'), - ... - IPNetwork('0:0:8000::/33')] - - Args: - other: An IPvXNetwork object of the same type. - - Returns: - A sorted list of IPvXNetwork objects addresses which is self - minus other. - - Raises: - TypeError: If self and other are of difffering address - versions, or if other is not a network object. - ValueError: If other is not completely contained by self. - - """ - if not self._version == other._version: - raise TypeError("%s and %s are not of the same version" % ( - str(self), str(other))) - - if not isinstance(other, _BaseNet): - raise TypeError("%s is not a network object" % str(other)) - - if other not in self: - raise ValueError('%s not contained in %s' % (str(other), - str(self))) - if other == self: - return [] - - ret_addrs = [] - - # Make sure we're comparing the network of other. - other = IPNetwork('%s/%s' % (str(other.network), str(other.prefixlen)), - version=other._version) - - s1, s2 = self.subnet() - while s1 != other and s2 != other: - if other in s1: - ret_addrs.append(s2) - s1, s2 = s1.subnet() - elif other in s2: - ret_addrs.append(s1) - s1, s2 = s2.subnet() - else: - # If we got here, there's a bug somewhere. - assert True == False, ('Error performing exclusion: ' - 's1: %s s2: %s other: %s' % - (str(s1), str(s2), str(other))) - if s1 == other: - ret_addrs.append(s2) - elif s2 == other: - ret_addrs.append(s1) - else: - # If we got here, there's a bug somewhere. - assert True == False, ('Error performing exclusion: ' - 's1: %s s2: %s other: %s' % - (str(s1), str(s2), str(other))) - - return sorted(ret_addrs, key=_BaseNet._get_networks_key) - - def compare_networks(self, other): - """Compare two IP objects. - - This is only concerned about the comparison of the integer - representation of the network addresses. This means that the - host bits aren't considered at all in this method. If you want - to compare host bits, you can easily enough do a - 'HostA._ip < HostB._ip' - - Args: - other: An IP object. - - Returns: - If the IP versions of self and other are the same, returns: - - -1 if self < other: - eg: IPv4('1.1.1.0/24') < IPv4('1.1.2.0/24') - IPv6('1080::200C:417A') < IPv6('1080::200B:417B') - 0 if self == other - eg: IPv4('1.1.1.1/24') == IPv4('1.1.1.2/24') - IPv6('1080::200C:417A/96') == IPv6('1080::200C:417B/96') - 1 if self > other - eg: IPv4('1.1.1.0/24') > IPv4('1.1.0.0/24') - IPv6('1080::1:200C:417A/112') > - IPv6('1080::0:200C:417A/112') - - If the IP versions of self and other are different, returns: - - -1 if self._version < other._version - eg: IPv4('10.0.0.1/24') < IPv6('::1/128') - 1 if self._version > other._version - eg: IPv6('::1/128') > IPv4('255.255.255.0/24') - - """ - if self._version < other._version: - return -1 - if self._version > other._version: - return 1 - # self._version == other._version below here: - if self.network < other.network: - return -1 - if self.network > other.network: - return 1 - # self.network == other.network below here: - if self.netmask < other.netmask: - return -1 - if self.netmask > other.netmask: - return 1 - # self.network == other.network and self.netmask == other.netmask - return 0 - - def _get_networks_key(self): - """Network-only key function. - - Returns an object that identifies this address' network and - netmask. This function is a suitable "key" argument for sorted() - and list.sort(). - - """ - return (self._version, self.network, self.netmask) - - def _ip_int_from_prefix(self, prefixlen=None): - """Turn the prefix length netmask into a int for comparison. - - Args: - prefixlen: An integer, the prefix length. - - Returns: - An integer. - - """ - if not prefixlen and prefixlen != 0: - prefixlen = self._prefixlen - return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen) - - def _prefix_from_ip_int(self, ip_int, mask=32): - """Return prefix length from the decimal netmask. - - Args: - ip_int: An integer, the IP address. - mask: The netmask. Defaults to 32. - - Returns: - An integer, the prefix length. - - """ - while mask: - if ip_int & 1 == 1: - break - ip_int >>= 1 - mask -= 1 - - return mask - - def _ip_string_from_prefix(self, prefixlen=None): - """Turn a prefix length into a dotted decimal string. - - Args: - prefixlen: An integer, the netmask prefix length. - - Returns: - A string, the dotted decimal netmask string. - - """ - if not prefixlen: - prefixlen = self._prefixlen - return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen)) - - def iter_subnets(self, prefixlen_diff=1, new_prefix=None): - """The subnets which join to make the current subnet. - - In the case that self contains only one IP - (self._prefixlen == 32 for IPv4 or self._prefixlen == 128 - for IPv6), return a list with just ourself. - - Args: - prefixlen_diff: An integer, the amount the prefix length - should be increased by. This should not be set if - new_prefix is also set. - new_prefix: The desired new prefix length. This must be a - larger number (smaller prefix) than the existing prefix. - This should not be set if prefixlen_diff is also set. - - Returns: - An iterator of IPv(4|6) objects. - - Raises: - ValueError: The prefixlen_diff is too small or too large. - OR - prefixlen_diff and new_prefix are both set or new_prefix - is a smaller number than the current prefix (smaller - number means a larger network) - - """ - if self._prefixlen == self._max_prefixlen: - yield self - return - - if new_prefix is not None: - if new_prefix < self._prefixlen: - raise ValueError('new prefix must be longer') - if prefixlen_diff != 1: - raise ValueError('cannot set prefixlen_diff and new_prefix') - prefixlen_diff = new_prefix - self._prefixlen - - if prefixlen_diff < 0: - raise ValueError('prefix length diff must be > 0') - new_prefixlen = self._prefixlen + prefixlen_diff - - if not self._is_valid_netmask(str(new_prefixlen)): - raise ValueError( - 'prefix length diff %d is invalid for netblock %s' % ( - new_prefixlen, str(self))) - - first = IPNetwork('%s/%s' % (str(self.network), - str(self._prefixlen + prefixlen_diff)), - version=self._version) - - yield first - current = first - while True: - broadcast = current.broadcast - if broadcast == self.broadcast: - return - new_addr = IPAddress(int(broadcast) + 1, version=self._version) - current = IPNetwork('%s/%s' % (str(new_addr), str(new_prefixlen)), - version=self._version) - - yield current - - def masked(self): - """Return the network object with the host bits masked out.""" - return IPNetwork('%s/%d' % (self.network, self._prefixlen), - version=self._version) - - def subnet(self, prefixlen_diff=1, new_prefix=None): - """Return a list of subnets, rather than an iterator.""" - return list(self.iter_subnets(prefixlen_diff, new_prefix)) - - def supernet(self, prefixlen_diff=1, new_prefix=None): - """The supernet containing the current network. - - Args: - prefixlen_diff: An integer, the amount the prefix length of - the network should be decreased by. For example, given a - /24 network and a prefixlen_diff of 3, a supernet with a - /21 netmask is returned. - - Returns: - An IPv4 network object. - - Raises: - ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have a - negative prefix length. - OR - If prefixlen_diff and new_prefix are both set or new_prefix is a - larger number than the current prefix (larger number means a - smaller network) - - """ - if self._prefixlen == 0: - return self - - if new_prefix is not None: - if new_prefix > self._prefixlen: - raise ValueError('new prefix must be shorter') - if prefixlen_diff != 1: - raise ValueError('cannot set prefixlen_diff and new_prefix') - prefixlen_diff = self._prefixlen - new_prefix - - - if self.prefixlen - prefixlen_diff < 0: - raise ValueError( - 'current prefixlen is %d, cannot have a prefixlen_diff of %d' % - (self.prefixlen, prefixlen_diff)) - return IPNetwork('%s/%s' % (str(self.network), - str(self.prefixlen - prefixlen_diff)), - version=self._version) - - # backwards compatibility - Subnet = subnet - Supernet = supernet - AddressExclude = address_exclude - CompareNetworks = compare_networks - Contains = __contains__ - - -class _BaseV4(object): - - """Base IPv4 object. - - The following methods are used by IPv4 objects in both single IP - addresses and networks. - - """ - - # Equivalent to 255.255.255.255 or 32 bits of 1's. - _ALL_ONES = (2**IPV4LENGTH) - 1 - _DECIMAL_DIGITS = frozenset('0123456789') - - def __init__(self, address): - self._version = 4 - self._max_prefixlen = IPV4LENGTH - - def _explode_shorthand_ip_string(self): - return str(self) - - def _ip_int_from_string(self, ip_str): - """Turn the given IP string into an integer for comparison. - - Args: - ip_str: A string, the IP ip_str. - - Returns: - The IP ip_str as an integer. - - Raises: - AddressValueError: if ip_str isn't a valid IPv4 Address. - - """ - octets = ip_str.split('.') - if len(octets) != 4: - raise AddressValueError(ip_str) - - packed_ip = 0 - for oc in octets: - try: - packed_ip = (packed_ip << 8) | self._parse_octet(oc) - except ValueError: - raise AddressValueError(ip_str) - return packed_ip - - def _parse_octet(self, octet_str): - """Convert a decimal octet into an integer. - - Args: - octet_str: A string, the number to parse. - - Returns: - The octet as an integer. - - Raises: - ValueError: if the octet isn't strictly a decimal from [0..255]. - - """ - # Whitelist the characters, since int() allows a lot of bizarre stuff. - if not self._DECIMAL_DIGITS.issuperset(octet_str): - raise ValueError - octet_int = int(octet_str, 10) - # Disallow leading zeroes, because no clear standard exists on - # whether these should be interpreted as decimal or octal. - if octet_int > 255 or (octet_str[0] == '0' and len(octet_str) > 1): - raise ValueError - return octet_int - - def _string_from_ip_int(self, ip_int): - """Turns a 32-bit integer into dotted decimal notation. - - Args: - ip_int: An integer, the IP address. - - Returns: - The IP address as a string in dotted decimal notation. - - """ - octets = [] - for _ in xrange(4): - octets.insert(0, str(ip_int & 0xFF)) - ip_int >>= 8 - return '.'.join(octets) - - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def packed(self): - """The binary representation of this address.""" - return v4_int_to_packed(self._ip) - - @property - def version(self): - return self._version - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within the - reserved IPv4 Network range. - - """ - return self in IPv4Network('240.0.0.0/4') - - @property - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per RFC 1918. - - """ - return (self in IPv4Network('10.0.0.0/8') or - self in IPv4Network('172.16.0.0/12') or - self in IPv4Network('192.168.0.0/16')) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is multicast. - See RFC 3171 for details. - - """ - return self in IPv4Network('224.0.0.0/4') - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 5735 3. - - """ - return self in IPv4Network('0.0.0.0') - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback per RFC 3330. - - """ - return self in IPv4Network('127.0.0.0/8') - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is link-local per RFC 3927. - - """ - return self in IPv4Network('169.254.0.0/16') - - -class IPv4Address(_BaseV4, _BaseIP): - - """Represent and manipulate single IPv4 Addresses.""" - - def __init__(self, address): - - """ - Args: - address: A string or integer representing the IP - '192.168.1.1' - - Additionally, an integer can be passed, so - IPv4Address('192.168.1.1') == IPv4Address(3232235777). - or, more generally - IPv4Address(int(IPv4Address('192.168.1.1'))) == - IPv4Address('192.168.1.1') - - Raises: - AddressValueError: If ipaddr isn't a valid IPv4 address. - - """ - _BaseV4.__init__(self, address) - - # Efficient constructor from integer. - if isinstance(address, (int, long)): - self._ip = address - if address < 0 or address > self._ALL_ONES: - raise AddressValueError(address) - return - - # Constructing from a packed address - if isinstance(address, Bytes): - try: - self._ip, = struct.unpack('!I', address) - except struct.error: - raise AddressValueError(address) # Wrong length. - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP string. - addr_str = str(address) - self._ip = self._ip_int_from_string(addr_str) - - -class IPv4Network(_BaseV4, _BaseNet): - - """This class represents and manipulates 32-bit IPv4 networks. - - Attributes: [examples for IPv4Network('1.2.3.4/27')] - ._ip: 16909060 - .ip: IPv4Address('1.2.3.4') - .network: IPv4Address('1.2.3.0') - .hostmask: IPv4Address('0.0.0.31') - .broadcast: IPv4Address('1.2.3.31') - .netmask: IPv4Address('255.255.255.224') - .prefixlen: 27 - - """ - - # the valid octets for host and netmasks. only useful for IPv4. - _valid_mask_octets = set((255, 254, 252, 248, 240, 224, 192, 128, 0)) - - def __init__(self, address, strict=False): - """Instantiate a new IPv4 network object. - - Args: - address: A string or integer representing the IP [& network]. - '192.168.1.1/24' - '192.168.1.1/255.255.255.0' - '192.168.1.1/0.0.0.255' - are all functionally the same in IPv4. Similarly, - '192.168.1.1' - '192.168.1.1/255.255.255.255' - '192.168.1.1/32' - are also functionaly equivalent. That is to say, failing to - provide a subnetmask will create an object with a mask of /32. - - If the mask (portion after the / in the argument) is given in - dotted quad form, it is treated as a netmask if it starts with a - non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it - starts with a zero field (e.g. 0.255.255.255 == /8), with the - single exception of an all-zero mask which is treated as a - netmask == /0. If no mask is given, a default of /32 is used. - - Additionally, an integer can be passed, so - IPv4Network('192.168.1.1') == IPv4Network(3232235777). - or, more generally - IPv4Network(int(IPv4Network('192.168.1.1'))) == - IPv4Network('192.168.1.1') - - strict: A boolean. If true, ensure that we have been passed - A true network address, eg, 192.168.1.0/24 and not an - IP address on a network, eg, 192.168.1.1/24. - - Raises: - AddressValueError: If ipaddr isn't a valid IPv4 address. - NetmaskValueError: If the netmask isn't valid for - an IPv4 address. - ValueError: If strict was True and a network address was not - supplied. - - """ - _BaseNet.__init__(self, address) - _BaseV4.__init__(self, address) - - # Constructing from an integer or packed bytes. - if isinstance(address, (int, long, Bytes)): - self.ip = IPv4Address(address) - self._ip = self.ip._ip - self._prefixlen = self._max_prefixlen - self.netmask = IPv4Address(self._ALL_ONES) - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - addr = str(address).split('/') - - if len(addr) > 2: - raise AddressValueError(address) - - self._ip = self._ip_int_from_string(addr[0]) - self.ip = IPv4Address(self._ip) - - if len(addr) == 2: - mask = addr[1].split('.') - if len(mask) == 4: - # We have dotted decimal netmask. - if self._is_valid_netmask(addr[1]): - self.netmask = IPv4Address(self._ip_int_from_string( - addr[1])) - elif self._is_hostmask(addr[1]): - self.netmask = IPv4Address( - self._ip_int_from_string(addr[1]) ^ self._ALL_ONES) - else: - raise NetmaskValueError('%s is not a valid netmask' - % addr[1]) - - self._prefixlen = self._prefix_from_ip_int(int(self.netmask)) - else: - # We have a netmask in prefix length form. - if not self._is_valid_netmask(addr[1]): - raise NetmaskValueError(addr[1]) - self._prefixlen = int(addr[1]) - self.netmask = IPv4Address(self._ip_int_from_prefix( - self._prefixlen)) - else: - self._prefixlen = self._max_prefixlen - self.netmask = IPv4Address(self._ip_int_from_prefix( - self._prefixlen)) - if strict: - if self.ip != self.network: - raise ValueError('%s has host bits set' % - self.ip) - if self._prefixlen == (self._max_prefixlen - 1): - self.iterhosts = self.__iter__ - - def _is_hostmask(self, ip_str): - """Test if the IP string is a hostmask (rather than a netmask). - - Args: - ip_str: A string, the potential hostmask. - - Returns: - A boolean, True if the IP string is a hostmask. - - """ - bits = ip_str.split('.') - try: - parts = [int(x) for x in bits if int(x) in self._valid_mask_octets] - except ValueError: - return False - if len(parts) != len(bits): - return False - if parts[0] < parts[-1]: - return True - return False - - def _is_valid_netmask(self, netmask): - """Verify that the netmask is valid. - - Args: - netmask: A string, either a prefix or dotted decimal - netmask. - - Returns: - A boolean, True if the prefix represents a valid IPv4 - netmask. - - """ - mask = netmask.split('.') - if len(mask) == 4: - if [x for x in mask if int(x) not in self._valid_mask_octets]: - return False - if [y for idx, y in enumerate(mask) if idx > 0 and - y > mask[idx - 1]]: - return False - return True - try: - netmask = int(netmask) - except ValueError: - return False - return 0 <= netmask <= self._max_prefixlen - - # backwards compatibility - IsRFC1918 = lambda self: self.is_private - IsMulticast = lambda self: self.is_multicast - IsLoopback = lambda self: self.is_loopback - IsLinkLocal = lambda self: self.is_link_local - - -class _BaseV6(object): - - """Base IPv6 object. - - The following methods are used by IPv6 objects in both single IP - addresses and networks. - - """ - - _ALL_ONES = (2**IPV6LENGTH) - 1 - _HEXTET_COUNT = 8 - _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - - def __init__(self, address): - self._version = 6 - self._max_prefixlen = IPV6LENGTH - - def _ip_int_from_string(self, ip_str): - """Turn an IPv6 ip_str into an integer. - - Args: - ip_str: A string, the IPv6 ip_str. - - Returns: - A long, the IPv6 ip_str. - - Raises: - AddressValueError: if ip_str isn't a valid IPv6 Address. - - """ - parts = ip_str.split(':') - - # An IPv6 address needs at least 2 colons (3 parts). - if len(parts) < 3: - raise AddressValueError(ip_str) - - # If the address has an IPv4-style suffix, convert it to hexadecimal. - if '.' in parts[-1]: - ipv4_int = IPv4Address(parts.pop())._ip - parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF)) - parts.append('%x' % (ipv4_int & 0xFFFF)) - - # An IPv6 address can't have more than 8 colons (9 parts). - if len(parts) > self._HEXTET_COUNT + 1: - raise AddressValueError(ip_str) - - # Disregarding the endpoints, find '::' with nothing in between. - # This indicates that a run of zeroes has been skipped. - try: - skip_index, = ( - [i for i in xrange(1, len(parts) - 1) if not parts[i]] or - [None]) - except ValueError: - # Can't have more than one '::' - raise AddressValueError(ip_str) - - # parts_hi is the number of parts to copy from above/before the '::' - # parts_lo is the number of parts to copy from below/after the '::' - if skip_index is not None: - # If we found a '::', then check if it also covers the endpoints. - parts_hi = skip_index - parts_lo = len(parts) - skip_index - 1 - if not parts[0]: - parts_hi -= 1 - if parts_hi: - raise AddressValueError(ip_str) # ^: requires ^:: - if not parts[-1]: - parts_lo -= 1 - if parts_lo: - raise AddressValueError(ip_str) # :$ requires ::$ - parts_skipped = self._HEXTET_COUNT - (parts_hi + parts_lo) - if parts_skipped < 1: - raise AddressValueError(ip_str) - else: - # Otherwise, allocate the entire address to parts_hi. The endpoints - # could still be empty, but _parse_hextet() will check for that. - if len(parts) != self._HEXTET_COUNT: - raise AddressValueError(ip_str) - parts_hi = len(parts) - parts_lo = 0 - parts_skipped = 0 - - try: - # Now, parse the hextets into a 128-bit integer. - ip_int = 0L - for i in xrange(parts_hi): - ip_int <<= 16 - ip_int |= self._parse_hextet(parts[i]) - ip_int <<= 16 * parts_skipped - for i in xrange(-parts_lo, 0): - ip_int <<= 16 - ip_int |= self._parse_hextet(parts[i]) - return ip_int - except ValueError: - raise AddressValueError(ip_str) - - def _parse_hextet(self, hextet_str): - """Convert an IPv6 hextet string into an integer. - - Args: - hextet_str: A string, the number to parse. - - Returns: - The hextet as an integer. - - Raises: - ValueError: if the input isn't strictly a hex number from [0..FFFF]. - - """ - # Whitelist the characters, since int() allows a lot of bizarre stuff. - if not self._HEX_DIGITS.issuperset(hextet_str): - raise ValueError - hextet_int = int(hextet_str, 16) - if hextet_int > 0xFFFF: - raise ValueError - return hextet_int - - def _compress_hextets(self, hextets): - """Compresses a list of hextets. - - Compresses a list of strings, replacing the longest continuous - sequence of "0" in the list with "" and adding empty strings at - the beginning or at the end of the string such that subsequently - calling ":".join(hextets) will produce the compressed version of - the IPv6 address. - - Args: - hextets: A list of strings, the hextets to compress. - - Returns: - A list of strings. - - """ - best_doublecolon_start = -1 - best_doublecolon_len = 0 - doublecolon_start = -1 - doublecolon_len = 0 - for index in range(len(hextets)): - if hextets[index] == '0': - doublecolon_len += 1 - if doublecolon_start == -1: - # Start of a sequence of zeros. - doublecolon_start = index - if doublecolon_len > best_doublecolon_len: - # This is the longest sequence of zeros so far. - best_doublecolon_len = doublecolon_len - best_doublecolon_start = doublecolon_start - else: - doublecolon_len = 0 - doublecolon_start = -1 - - if best_doublecolon_len > 1: - best_doublecolon_end = (best_doublecolon_start + - best_doublecolon_len) - # For zeros at the end of the address. - if best_doublecolon_end == len(hextets): - hextets += [''] - hextets[best_doublecolon_start:best_doublecolon_end] = [''] - # For zeros at the beginning of the address. - if best_doublecolon_start == 0: - hextets = [''] + hextets - - return hextets - - def _string_from_ip_int(self, ip_int=None): - """Turns a 128-bit integer into hexadecimal notation. - - Args: - ip_int: An integer, the IP address. - - Returns: - A string, the hexadecimal representation of the address. - - Raises: - ValueError: The address is bigger than 128 bits of all ones. - - """ - if not ip_int and ip_int != 0: - ip_int = int(self._ip) - - if ip_int > self._ALL_ONES: - raise ValueError('IPv6 address is too large') - - hex_str = '%032x' % ip_int - hextets = [] - for x in range(0, 32, 4): - hextets.append('%x' % int(hex_str[x:x+4], 16)) - - hextets = self._compress_hextets(hextets) - return ':'.join(hextets) - - def _explode_shorthand_ip_string(self): - """Expand a shortened IPv6 address. - - Args: - ip_str: A string, the IPv6 address. - - Returns: - A string, the expanded IPv6 address. - - """ - if isinstance(self, _BaseNet): - ip_str = str(self.ip) - else: - ip_str = str(self) - - ip_int = self._ip_int_from_string(ip_str) - parts = [] - for i in xrange(self._HEXTET_COUNT): - parts.append('%04x' % (ip_int & 0xFFFF)) - ip_int >>= 16 - parts.reverse() - if isinstance(self, _BaseNet): - return '%s/%d' % (':'.join(parts), self.prefixlen) - return ':'.join(parts) - - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def packed(self): - """The binary representation of this address.""" - return v6_int_to_packed(self._ip) - - @property - def version(self): - return self._version - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is a multicast address. - See RFC 2373 2.7 for details. - - """ - return self in IPv6Network('ff00::/8') - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within one of the - reserved IPv6 Network ranges. - - """ - return (self in IPv6Network('::/8') or - self in IPv6Network('100::/8') or - self in IPv6Network('200::/7') or - self in IPv6Network('400::/6') or - self in IPv6Network('800::/5') or - self in IPv6Network('1000::/4') or - self in IPv6Network('4000::/3') or - self in IPv6Network('6000::/3') or - self in IPv6Network('8000::/3') or - self in IPv6Network('A000::/3') or - self in IPv6Network('C000::/3') or - self in IPv6Network('E000::/4') or - self in IPv6Network('F000::/5') or - self in IPv6Network('F800::/6') or - self in IPv6Network('FE00::/9')) - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 2373 2.5.2. - - """ - return self._ip == 0 and getattr(self, '_prefixlen', 128) == 128 - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback address as defined in - RFC 2373 2.5.3. - - """ - return self._ip == 1 and getattr(self, '_prefixlen', 128) == 128 - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is reserved per RFC 4291. - - """ - return self in IPv6Network('fe80::/10') - - @property - def is_site_local(self): - """Test if the address is reserved for site-local. - - Note that the site-local address space has been deprecated by RFC 3879. - Use is_private to test if this address is in the space of unique local - addresses as defined by RFC 4193. - - Returns: - A boolean, True if the address is reserved per RFC 3513 2.5.6. - - """ - return self in IPv6Network('fec0::/10') - - @property - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per RFC 4193. - - """ - return self in IPv6Network('fc00::/7') - - @property - def ipv4_mapped(self): - """Return the IPv4 mapped address. - - Returns: - If the IPv6 address is a v4 mapped address, return the - IPv4 mapped address. Return None otherwise. - - """ - if (self._ip >> 32) != 0xFFFF: - return None - return IPv4Address(self._ip & 0xFFFFFFFF) - - @property - def teredo(self): - """Tuple of embedded teredo IPs. - - Returns: - Tuple of the (server, client) IPs or None if the address - doesn't appear to be a teredo address (doesn't start with - 2001::/32) - - """ - if (self._ip >> 96) != 0x20010000: - return None - return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF), - IPv4Address(~self._ip & 0xFFFFFFFF)) - - @property - def sixtofour(self): - """Return the IPv4 6to4 embedded address. - - Returns: - The IPv4 6to4-embedded address if present or None if the - address doesn't appear to contain a 6to4 embedded address. - - """ - if (self._ip >> 112) != 0x2002: - return None - return IPv4Address((self._ip >> 80) & 0xFFFFFFFF) - - -class IPv6Address(_BaseV6, _BaseIP): - - """Represent and manipulate single IPv6 Addresses. - """ - - def __init__(self, address): - """Instantiate a new IPv6 address object. - - Args: - address: A string or integer representing the IP - - Additionally, an integer can be passed, so - IPv6Address('2001:4860::') == - IPv6Address(42541956101370907050197289607612071936L). - or, more generally - IPv6Address(IPv6Address('2001:4860::')._ip) == - IPv6Address('2001:4860::') - - Raises: - AddressValueError: If address isn't a valid IPv6 address. - - """ - _BaseV6.__init__(self, address) - - # Efficient constructor from integer. - if isinstance(address, (int, long)): - self._ip = address - if address < 0 or address > self._ALL_ONES: - raise AddressValueError(address) - return - - # Constructing from a packed address - if isinstance(address, Bytes): - try: - hi, lo = struct.unpack('!QQ', address) - except struct.error: - raise AddressValueError(address) # Wrong length. - self._ip = (hi << 64) | lo - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP string. - addr_str = str(address) - if not addr_str: - raise AddressValueError('') - - self._ip = self._ip_int_from_string(addr_str) - - -class IPv6Network(_BaseV6, _BaseNet): - - """This class represents and manipulates 128-bit IPv6 networks. - - Attributes: [examples for IPv6('2001:658:22A:CAFE:200::1/64')] - .ip: IPv6Address('2001:658:22a:cafe:200::1') - .network: IPv6Address('2001:658:22a:cafe::') - .hostmask: IPv6Address('::ffff:ffff:ffff:ffff') - .broadcast: IPv6Address('2001:658:22a:cafe:ffff:ffff:ffff:ffff') - .netmask: IPv6Address('ffff:ffff:ffff:ffff::') - .prefixlen: 64 - - """ - - - def __init__(self, address, strict=False): - """Instantiate a new IPv6 Network object. - - Args: - address: A string or integer representing the IPv6 network or the IP - and prefix/netmask. - '2001:4860::/128' - '2001:4860:0000:0000:0000:0000:0000:0000/128' - '2001:4860::' - are all functionally the same in IPv6. That is to say, - failing to provide a subnetmask will create an object with - a mask of /128. - - Additionally, an integer can be passed, so - IPv6Network('2001:4860::') == - IPv6Network(42541956101370907050197289607612071936L). - or, more generally - IPv6Network(IPv6Network('2001:4860::')._ip) == - IPv6Network('2001:4860::') - - strict: A boolean. If true, ensure that we have been passed - A true network address, eg, 192.168.1.0/24 and not an - IP address on a network, eg, 192.168.1.1/24. - - Raises: - AddressValueError: If address isn't a valid IPv6 address. - NetmaskValueError: If the netmask isn't valid for - an IPv6 address. - ValueError: If strict was True and a network address was not - supplied. - - """ - _BaseNet.__init__(self, address) - _BaseV6.__init__(self, address) - - # Constructing from an integer or packed bytes. - if isinstance(address, (int, long, Bytes)): - self.ip = IPv6Address(address) - self._ip = self.ip._ip - self._prefixlen = self._max_prefixlen - self.netmask = IPv6Address(self._ALL_ONES) - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - addr = str(address).split('/') - - if len(addr) > 2: - raise AddressValueError(address) - - self._ip = self._ip_int_from_string(addr[0]) - self.ip = IPv6Address(self._ip) - - if len(addr) == 2: - if self._is_valid_netmask(addr[1]): - self._prefixlen = int(addr[1]) - else: - raise NetmaskValueError(addr[1]) - else: - self._prefixlen = self._max_prefixlen - - self.netmask = IPv6Address(self._ip_int_from_prefix(self._prefixlen)) - - if strict: - if self.ip != self.network: - raise ValueError('%s has host bits set' % - self.ip) - if self._prefixlen == (self._max_prefixlen - 1): - self.iterhosts = self.__iter__ - - def _is_valid_netmask(self, prefixlen): - """Verify that the netmask/prefixlen is valid. - - Args: - prefixlen: A string, the netmask in prefix length format. - - Returns: - A boolean, True if the prefix represents a valid IPv6 - netmask. - - """ - try: - prefixlen = int(prefixlen) - except ValueError: - return False - return 0 <= prefixlen <= self._max_prefixlen - - @property - def with_netmask(self): - return self.with_prefixlen diff --git a/plugins/Sidebar/maxminddb/reader.py b/plugins/Sidebar/maxminddb/reader.py deleted file mode 100644 index 5ecfbdf2..00000000 --- a/plugins/Sidebar/maxminddb/reader.py +++ /dev/null @@ -1,221 +0,0 @@ -""" -maxminddb.reader -~~~~~~~~~~~~~~~~ - -This module contains the pure Python database reader and related classes. - -""" -from __future__ import unicode_literals - -try: - import mmap -except ImportError: - # pylint: disable=invalid-name - mmap = None - -import struct - -from maxminddb.compat import byte_from_int, int_from_byte, ipaddress -from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY -from maxminddb.decoder import Decoder -from maxminddb.errors import InvalidDatabaseError -from maxminddb.file import FileBuffer - - -class Reader(object): - - """ - Instances of this class provide a reader for the MaxMind DB format. IP - addresses can be looked up using the ``get`` method. - """ - - _DATA_SECTION_SEPARATOR_SIZE = 16 - _METADATA_START_MARKER = b"\xAB\xCD\xEFMaxMind.com" - - _ipv4_start = None - - def __init__(self, database, mode=MODE_AUTO): - """Reader for the MaxMind DB file format - - Arguments: - database -- A path to a valid MaxMind DB file such as a GeoIP2 - database file. - mode -- mode to open the database with. Valid mode are: - * MODE_MMAP - read from memory map. - * MODE_FILE - read database as standard file. - * MODE_MEMORY - load database into memory. - * MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default. - """ - if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: - with open(database, 'rb') as db_file: - self._buffer = mmap.mmap( - db_file.fileno(), 0, access=mmap.ACCESS_READ) - self._buffer_size = self._buffer.size() - elif mode in (MODE_AUTO, MODE_FILE): - self._buffer = FileBuffer(database) - self._buffer_size = self._buffer.size() - elif mode == MODE_MEMORY: - with open(database, 'rb') as db_file: - self._buffer = db_file.read() - self._buffer_size = len(self._buffer) - else: - raise ValueError('Unsupported open mode ({0}). Only MODE_AUTO, ' - ' MODE_FILE, and MODE_MEMORY are support by the pure Python ' - 'Reader'.format(mode)) - - metadata_start = self._buffer.rfind(self._METADATA_START_MARKER, - max(0, self._buffer_size - - 128 * 1024)) - - if metadata_start == -1: - self.close() - raise InvalidDatabaseError('Error opening database file ({0}). ' - 'Is this a valid MaxMind DB file?' - ''.format(database)) - - metadata_start += len(self._METADATA_START_MARKER) - metadata_decoder = Decoder(self._buffer, metadata_start) - (metadata, _) = metadata_decoder.decode(metadata_start) - self._metadata = Metadata( - **metadata) # pylint: disable=bad-option-value - - self._decoder = Decoder(self._buffer, self._metadata.search_tree_size - + self._DATA_SECTION_SEPARATOR_SIZE) - - def metadata(self): - """Return the metadata associated with the MaxMind DB file""" - return self._metadata - - def get(self, ip_address): - """Return the record for the ip_address in the MaxMind DB - - - Arguments: - ip_address -- an IP address in the standard string notation - """ - address = ipaddress.ip_address(ip_address) - - if address.version == 6 and self._metadata.ip_version == 4: - raise ValueError('Error looking up {0}. You attempted to look up ' - 'an IPv6 address in an IPv4-only database.'.format( - ip_address)) - pointer = self._find_address_in_tree(address) - - return self._resolve_data_pointer(pointer) if pointer else None - - def _find_address_in_tree(self, ip_address): - packed = ip_address.packed - - bit_count = len(packed) * 8 - node = self._start_node(bit_count) - - for i in range(bit_count): - if node >= self._metadata.node_count: - break - bit = 1 & (int_from_byte(packed[i >> 3]) >> 7 - (i % 8)) - node = self._read_node(node, bit) - if node == self._metadata.node_count: - # Record is empty - return 0 - elif node > self._metadata.node_count: - return node - - raise InvalidDatabaseError('Invalid node in search tree') - - def _start_node(self, length): - if self._metadata.ip_version != 6 or length == 128: - return 0 - - # We are looking up an IPv4 address in an IPv6 tree. Skip over the - # first 96 nodes. - if self._ipv4_start: - return self._ipv4_start - - node = 0 - for _ in range(96): - if node >= self._metadata.node_count: - break - node = self._read_node(node, 0) - self._ipv4_start = node - return node - - def _read_node(self, node_number, index): - base_offset = node_number * self._metadata.node_byte_size - - record_size = self._metadata.record_size - if record_size == 24: - offset = base_offset + index * 3 - node_bytes = b'\x00' + self._buffer[offset:offset + 3] - elif record_size == 28: - (middle,) = struct.unpack( - b'!B', self._buffer[base_offset + 3:base_offset + 4]) - if index: - middle &= 0x0F - else: - middle = (0xF0 & middle) >> 4 - offset = base_offset + index * 4 - node_bytes = byte_from_int( - middle) + self._buffer[offset:offset + 3] - elif record_size == 32: - offset = base_offset + index * 4 - node_bytes = self._buffer[offset:offset + 4] - else: - raise InvalidDatabaseError( - 'Unknown record size: {0}'.format(record_size)) - return struct.unpack(b'!I', node_bytes)[0] - - def _resolve_data_pointer(self, pointer): - resolved = pointer - self._metadata.node_count + \ - self._metadata.search_tree_size - - if resolved > self._buffer_size: - raise InvalidDatabaseError( - "The MaxMind DB file's search tree is corrupt") - - (data, _) = self._decoder.decode(resolved) - return data - - def close(self): - """Closes the MaxMind DB file and returns the resources to the system""" - # pylint: disable=unidiomatic-typecheck - if type(self._buffer) not in (str, bytes): - self._buffer.close() - - -class Metadata(object): - - """Metadata for the MaxMind DB reader""" - - # pylint: disable=too-many-instance-attributes - def __init__(self, **kwargs): - """Creates new Metadata object. kwargs are key/value pairs from spec""" - # Although I could just update __dict__, that is less obvious and it - # doesn't work well with static analysis tools and some IDEs - self.node_count = kwargs['node_count'] - self.record_size = kwargs['record_size'] - self.ip_version = kwargs['ip_version'] - self.database_type = kwargs['database_type'] - self.languages = kwargs['languages'] - self.binary_format_major_version = kwargs[ - 'binary_format_major_version'] - self.binary_format_minor_version = kwargs[ - 'binary_format_minor_version'] - self.build_epoch = kwargs['build_epoch'] - self.description = kwargs['description'] - - @property - def node_byte_size(self): - """The size of a node in bytes""" - return self.record_size // 4 - - @property - def search_tree_size(self): - """The size of the search tree""" - return self.node_count * self.node_byte_size - - def __repr__(self): - args = ', '.join('%s=%r' % x for x in self.__dict__.items()) - return '{module}.{class_name}({data})'.format( - module=self.__module__, - class_name=self.__class__.__name__, - data=args)