oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove included maxminddb

Browse source
This commit is contained in:
shortcutme 2019-03-16 02:52:12 +01:00
parent 567855e2d2
commit ff5004cb8d
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
9 changed files with 0 additions and 3018 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
plugins/Sidebar/maxminddb/__init__.py
View file

@ -1,46 +0,0 @@
# pylint:disable=C0111
import os
import maxminddb.reader
try:
import maxminddb.extension
except ImportError:
maxminddb.extension = None
from maxminddb.const import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE,
MODE_MEMORY)
from maxminddb.decoder import InvalidDatabaseError
def open_database(database, mode=MODE_AUTO):
"""Open a Maxmind DB database
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2
database file.
mode -- mode to open the database with. Valid mode are:
* MODE_MMAP_EXT - use the C extension with memory map.
* MODE_MMAP - read from memory map. Pure Python.
* MODE_FILE - read database as standard file. Pure Python.
* MODE_MEMORY - load database into memory. Pure Python.
* MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that
order. Default mode.
"""
if (mode == MODE_AUTO and maxminddb.extension and
hasattr(maxminddb.extension, 'Reader')) or mode == MODE_MMAP_EXT:
return maxminddb.extension.Reader(database)
elif mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY):
return maxminddb.reader.Reader(database, mode)
raise ValueError('Unsupported open mode: {0}'.format(mode))
def Reader(database): # pylint: disable=invalid-name
"""This exists for backwards compatibility. Use open_database instead"""
return open_database(database)
__title__ = 'maxminddb'
__version__ = '1.2.0'
__author__ = 'Gregory Oschwald'
__license__ = 'Apache License, Version 2.0'
__copyright__ = 'Copyright 2014 Maxmind, Inc.'

28
plugins/Sidebar/maxminddb/compat.py
View file

@ -1,28 +0,0 @@
import sys
# pylint: skip-file
if sys.version_info[0] == 2:
import ipaddr as ipaddress # pylint:disable=F0401
ipaddress.ip_address = ipaddress.IPAddress
int_from_byte = ord
FileNotFoundError = IOError
def int_from_bytes(b):
if b:
return int(b.encode("hex"), 16)
return 0
byte_from_int = chr
else:
import ipaddress # pylint:disable=F0401
int_from_byte = lambda x: x
FileNotFoundError = FileNotFoundError
int_from_bytes = lambda x: int.from_bytes(x, 'big')
byte_from_int = lambda x: bytes([x])

7
plugins/Sidebar/maxminddb/const.py
View file

@ -1,7 +0,0 @@
"""Constants used in the API"""
MODE_AUTO = 0
MODE_MMAP_EXT = 1
MODE_MMAP = 2
MODE_FILE = 4
MODE_MEMORY = 8

173
plugins/Sidebar/maxminddb/decoder.py
View file

@ -1,173 +0,0 @@
"""
maxminddb.decoder
~~~~~~~~~~~~~~~~~
This package contains code for decoding the MaxMind DB data section.
"""
from __future__ import unicode_literals
import struct
from maxminddb.compat import byte_from_int, int_from_bytes
from maxminddb.errors import InvalidDatabaseError
class Decoder(object): # pylint: disable=too-few-public-methods
"""Decoder for the data section of the MaxMind DB"""
def __init__(self, database_buffer, pointer_base=0, pointer_test=False):
"""Created a Decoder for a MaxMind DB
Arguments:
database_buffer -- an mmap'd MaxMind DB file.
pointer_base -- the base number to use when decoding a pointer
pointer_test -- used for internal unit testing of pointer code
"""
self._pointer_test = pointer_test
self._buffer = database_buffer
self._pointer_base = pointer_base
def _decode_array(self, size, offset):
array = []
for _ in range(size):
(value, offset) = self.decode(offset)
array.append(value)
return array, offset
def _decode_boolean(self, size, offset):
return size != 0, offset
def _decode_bytes(self, size, offset):
new_offset = offset + size
return self._buffer[offset:new_offset], new_offset
# pylint: disable=no-self-argument
# |-> I am open to better ways of doing this as long as it doesn't involve
# lots of code duplication.
def _decode_packed_type(type_code, type_size, pad=False):
# pylint: disable=protected-access, missing-docstring
def unpack_type(self, size, offset):
if not pad:
self._verify_size(size, type_size)
new_offset = offset + type_size
packed_bytes = self._buffer[offset:new_offset]
if pad:
packed_bytes = packed_bytes.rjust(type_size, b'\x00')
(value,) = struct.unpack(type_code, packed_bytes)
return value, new_offset
return unpack_type
def _decode_map(self, size, offset):
container = {}
for _ in range(size):
(key, offset) = self.decode(offset)
(value, offset) = self.decode(offset)
container[key] = value
return container, offset
_pointer_value_offset = {
1: 0,
2: 2048,
3: 526336,
4: 0,
}
def _decode_pointer(self, size, offset):
pointer_size = ((size >> 3) & 0x3) + 1
new_offset = offset + pointer_size
pointer_bytes = self._buffer[offset:new_offset]
packed = pointer_bytes if pointer_size == 4 else struct.pack(
b'!c', byte_from_int(size & 0x7)) + pointer_bytes
unpacked = int_from_bytes(packed)
pointer = unpacked + self._pointer_base + \
self._pointer_value_offset[pointer_size]
if self._pointer_test:
return pointer, new_offset
(value, _) = self.decode(pointer)
return value, new_offset
def _decode_uint(self, size, offset):
new_offset = offset + size
uint_bytes = self._buffer[offset:new_offset]
return int_from_bytes(uint_bytes), new_offset
def _decode_utf8_string(self, size, offset):
new_offset = offset + size
return self._buffer[offset:new_offset].decode('utf-8'), new_offset
_type_decoder = {
1: _decode_pointer,
2: _decode_utf8_string,
3: _decode_packed_type(b'!d', 8), # double,
4: _decode_bytes,
5: _decode_uint, # uint16
6: _decode_uint, # uint32
7: _decode_map,
8: _decode_packed_type(b'!i', 4, pad=True), # int32
9: _decode_uint, # uint64
10: _decode_uint, # uint128
11: _decode_array,
14: _decode_boolean,
15: _decode_packed_type(b'!f', 4), # float,
}
def decode(self, offset):
"""Decode a section of the data section starting at offset
Arguments:
offset -- the location of the data structure to decode
"""
new_offset = offset + 1
(ctrl_byte,) = struct.unpack(b'!B', self._buffer[offset:new_offset])
type_num = ctrl_byte >> 5
# Extended type
if not type_num:
(type_num, new_offset) = self._read_extended(new_offset)
if not type_num in self._type_decoder:
raise InvalidDatabaseError('Unexpected type number ({type}) '
'encountered'.format(type=type_num))
(size, new_offset) = self._size_from_ctrl_byte(
ctrl_byte, new_offset, type_num)
return self._type_decoder[type_num](self, size, new_offset)
def _read_extended(self, offset):
(next_byte,) = struct.unpack(b'!B', self._buffer[offset:offset + 1])
type_num = next_byte + 7
if type_num < 7:
raise InvalidDatabaseError(
'Something went horribly wrong in the decoder. An '
'extended type resolved to a type number < 8 '
'({type})'.format(type=type_num))
return type_num, offset + 1
def _verify_size(self, expected, actual):
if expected != actual:
raise InvalidDatabaseError(
'The MaxMind DB file\'s data section contains bad data '
'(unknown data type or corrupt data)'
)
def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num):
size = ctrl_byte & 0x1f
if type_num == 1:
return size, offset
bytes_to_read = 0 if size < 29 else size - 28
new_offset = offset + bytes_to_read
size_bytes = self._buffer[offset:new_offset]
# Using unpack rather than int_from_bytes as it is about 200 lookups
# per second faster here.
if size == 29:
size = 29 + struct.unpack(b'!B', size_bytes)[0]
elif size == 30:
size = 285 + struct.unpack(b'!H', size_bytes)[0]
elif size > 30:
size = struct.unpack(
b'!I', size_bytes.rjust(4, b'\x00'))[0] + 65821
return size, new_offset

11
plugins/Sidebar/maxminddb/errors.py
View file

@ -1,11 +0,0 @@
"""
maxminddb.errors
~~~~~~~~~~~~~~~~
This module contains custom errors for the MaxMind DB reader
"""
class InvalidDatabaseError(RuntimeError):
"""This error is thrown when unexpected data is found in the database."""

570
plugins/Sidebar/maxminddb/extension/maxminddb.c
View file

@ -1,570 +0,0 @@
#include <Python.h>
#include <maxminddb.h>
#include "structmember.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
static PyTypeObject Reader_Type;
static PyTypeObject Metadata_Type;
static PyObject *MaxMindDB_error;
typedef struct {
PyObject_HEAD /* no semicolon */
MMDB_s *mmdb;
} Reader_obj;
typedef struct {
PyObject_HEAD /* no semicolon */
PyObject *binary_format_major_version;
PyObject *binary_format_minor_version;
PyObject *build_epoch;
PyObject *database_type;
PyObject *description;
PyObject *ip_version;
PyObject *languages;
PyObject *node_count;
PyObject *record_size;
} Metadata_obj;
static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list);
static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list);
static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list);
static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list);
#if PY_MAJOR_VERSION >= 3
#define MOD_INIT(name) PyMODINIT_FUNC PyInit_ ## name(void)
#define RETURN_MOD_INIT(m) return (m)
#define FILE_NOT_FOUND_ERROR PyExc_FileNotFoundError
#else
#define MOD_INIT(name) PyMODINIT_FUNC init ## name(void)
#define RETURN_MOD_INIT(m) return
#define PyInt_FromLong PyLong_FromLong
#define FILE_NOT_FOUND_ERROR PyExc_IOError
#endif
#ifdef __GNUC__
# define UNUSED(x) UNUSED_ ## x __attribute__((__unused__))
#else
# define UNUSED(x) UNUSED_ ## x
#endif
static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds)
{
char *filename;
int mode = 0;
static char *kwlist[] = {"database", "mode", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", kwlist, &filename, &mode)) {
return -1;
}
if (mode != 0 && mode != 1) {
PyErr_Format(PyExc_ValueError, "Unsupported open mode (%i). Only "
"MODE_AUTO and MODE_MMAP_EXT are supported by this extension.",
mode);
return -1;
}
if (0 != access(filename, R_OK)) {
PyErr_Format(FILE_NOT_FOUND_ERROR,
"No such file or directory: '%s'",
filename);
return -1;
}
MMDB_s *mmdb = (MMDB_s *)malloc(sizeof(MMDB_s));
if (NULL == mmdb) {
PyErr_NoMemory();
return -1;
}
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (!mmdb_obj) {
free(mmdb);
PyErr_NoMemory();
return -1;
}
uint16_t status = MMDB_open(filename, MMDB_MODE_MMAP, mmdb);
if (MMDB_SUCCESS != status) {
free(mmdb);
PyErr_Format(
MaxMindDB_error,
"Error opening database file (%s). Is this a valid MaxMind DB file?",
filename
);
return -1;
}
mmdb_obj->mmdb = mmdb;
return 0;
}
static PyObject *Reader_get(PyObject *self, PyObject *args)
{
char *ip_address = NULL;
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (!PyArg_ParseTuple(args, "s", &ip_address)) {
return NULL;
}
MMDB_s *mmdb = mmdb_obj->mmdb;
if (NULL == mmdb) {
PyErr_SetString(PyExc_ValueError,
"Attempt to read from a closed MaxMind DB.");
return NULL;
}
int gai_error = 0;
int mmdb_error = MMDB_SUCCESS;
MMDB_lookup_result_s result =
MMDB_lookup_string(mmdb, ip_address, &gai_error,
&mmdb_error);
if (0 != gai_error) {
PyErr_Format(PyExc_ValueError,
"'%s' does not appear to be an IPv4 or IPv6 address.",
ip_address);
return NULL;
}
if (MMDB_SUCCESS != mmdb_error) {
PyObject *exception;
if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) {
exception = PyExc_ValueError;
} else {
exception = MaxMindDB_error;
}
PyErr_Format(exception, "Error looking up %s. %s",
ip_address, MMDB_strerror(mmdb_error));
return NULL;
}
if (!result.found_entry) {
Py_RETURN_NONE;
}
MMDB_entry_data_list_s *entry_data_list = NULL;
int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list);
if (MMDB_SUCCESS != status) {
PyErr_Format(MaxMindDB_error,
"Error while looking up data for %s. %s",
ip_address, MMDB_strerror(status));
MMDB_free_entry_data_list(entry_data_list);
return NULL;
}
MMDB_entry_data_list_s *original_entry_data_list = entry_data_list;
PyObject *py_obj = from_entry_data_list(&entry_data_list);
MMDB_free_entry_data_list(original_entry_data_list);
return py_obj;
}
static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args))
{
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (NULL == mmdb_obj->mmdb) {
PyErr_SetString(PyExc_IOError,
"Attempt to read from a closed MaxMind DB.");
return NULL;
}
MMDB_entry_data_list_s *entry_data_list;
MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list);
MMDB_entry_data_list_s *original_entry_data_list = entry_data_list;
PyObject *metadata_dict = from_entry_data_list(&entry_data_list);
MMDB_free_entry_data_list(original_entry_data_list);
if (NULL == metadata_dict || !PyDict_Check(metadata_dict)) {
PyErr_SetString(MaxMindDB_error,
"Error decoding metadata.");
return NULL;
}
PyObject *args = PyTuple_New(0);
if (NULL == args) {
Py_DECREF(metadata_dict);
return NULL;
}
PyObject *metadata = PyObject_Call((PyObject *)&Metadata_Type, args,
metadata_dict);
Py_DECREF(metadata_dict);
return metadata;
}
static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args))
{
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (NULL != mmdb_obj->mmdb) {
MMDB_close(mmdb_obj->mmdb);
free(mmdb_obj->mmdb);
mmdb_obj->mmdb = NULL;
}
Py_RETURN_NONE;
}
static void Reader_dealloc(PyObject *self)
{
Reader_obj *obj = (Reader_obj *)self;
if (NULL != obj->mmdb) {
Reader_close(self, NULL);
}
PyObject_Del(self);
}
static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject
*binary_format_major_version,
*binary_format_minor_version,
*build_epoch,
*database_type,
*description,
*ip_version,
*languages,
*node_count,
*record_size;
static char *kwlist[] = {
"binary_format_major_version",
"binary_format_minor_version",
"build_epoch",
"database_type",
"description",
"ip_version",
"languages",
"node_count",
"record_size",
NULL
};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOOOOOO", kwlist,
&binary_format_major_version,
&binary_format_minor_version,
&build_epoch,
&database_type,
&description,
&ip_version,
&languages,
&node_count,
&record_size)) {
return -1;
}
Metadata_obj *obj = (Metadata_obj *)self;
obj->binary_format_major_version = binary_format_major_version;
obj->binary_format_minor_version = binary_format_minor_version;
obj->build_epoch = build_epoch;
obj->database_type = database_type;
obj->description = description;
obj->ip_version = ip_version;
obj->languages = languages;
obj->node_count = node_count;
obj->record_size = record_size;
Py_INCREF(obj->binary_format_major_version);
Py_INCREF(obj->binary_format_minor_version);
Py_INCREF(obj->build_epoch);
Py_INCREF(obj->database_type);
Py_INCREF(obj->description);
Py_INCREF(obj->ip_version);
Py_INCREF(obj->languages);
Py_INCREF(obj->node_count);
Py_INCREF(obj->record_size);
return 0;
}
static void Metadata_dealloc(PyObject *self)
{
Metadata_obj *obj = (Metadata_obj *)self;
Py_DECREF(obj->binary_format_major_version);
Py_DECREF(obj->binary_format_minor_version);
Py_DECREF(obj->build_epoch);
Py_DECREF(obj->database_type);
Py_DECREF(obj->description);
Py_DECREF(obj->ip_version);
Py_DECREF(obj->languages);
Py_DECREF(obj->node_count);
Py_DECREF(obj->record_size);
PyObject_Del(self);
}
static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list)
{
if (NULL == entry_data_list || NULL == *entry_data_list) {
PyErr_SetString(
MaxMindDB_error,
"Error while looking up data. Your database may be corrupt or you have found a bug in libmaxminddb."
);
return NULL;
}
switch ((*entry_data_list)->entry_data.type) {
case MMDB_DATA_TYPE_MAP:
return from_map(entry_data_list);
case MMDB_DATA_TYPE_ARRAY:
return from_array(entry_data_list);
case MMDB_DATA_TYPE_UTF8_STRING:
return PyUnicode_FromStringAndSize(
(*entry_data_list)->entry_data.utf8_string,
(*entry_data_list)->entry_data.data_size
);
case MMDB_DATA_TYPE_BYTES:
return PyByteArray_FromStringAndSize(
(const char *)(*entry_data_list)->entry_data.bytes,
(Py_ssize_t)(*entry_data_list)->entry_data.data_size);
case MMDB_DATA_TYPE_DOUBLE:
return PyFloat_FromDouble((*entry_data_list)->entry_data.double_value);
case MMDB_DATA_TYPE_FLOAT:
return PyFloat_FromDouble((*entry_data_list)->entry_data.float_value);
case MMDB_DATA_TYPE_UINT16:
return PyLong_FromLong( (*entry_data_list)->entry_data.uint16);
case MMDB_DATA_TYPE_UINT32:
return PyLong_FromLong((*entry_data_list)->entry_data.uint32);
case MMDB_DATA_TYPE_BOOLEAN:
return PyBool_FromLong((*entry_data_list)->entry_data.boolean);
case MMDB_DATA_TYPE_UINT64:
return PyLong_FromUnsignedLongLong(
(*entry_data_list)->entry_data.uint64);
case MMDB_DATA_TYPE_UINT128:
return from_uint128(*entry_data_list);
case MMDB_DATA_TYPE_INT32:
return PyLong_FromLong((*entry_data_list)->entry_data.int32);
default:
PyErr_Format(MaxMindDB_error,
"Invalid data type arguments: %d",
(*entry_data_list)->entry_data.type);
return NULL;
}
return NULL;
}
static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list)
{
PyObject *py_obj = PyDict_New();
if (NULL == py_obj) {
PyErr_NoMemory();
return NULL;
}
const uint32_t map_size = (*entry_data_list)->entry_data.data_size;
uint i;
// entry_data_list cannot start out NULL (see from_entry_data_list). We
// check it in the loop because it may become NULL.
// coverity[check_after_deref]
for (i = 0; i < map_size && entry_data_list; i++) {
*entry_data_list = (*entry_data_list)->next;
PyObject *key = PyUnicode_FromStringAndSize(
(char *)(*entry_data_list)->entry_data.utf8_string,
(*entry_data_list)->entry_data.data_size
);
*entry_data_list = (*entry_data_list)->next;
PyObject *value = from_entry_data_list(entry_data_list);
if (NULL == value) {
Py_DECREF(key);
Py_DECREF(py_obj);
return NULL;
}
PyDict_SetItem(py_obj, key, value);
Py_DECREF(value);
Py_DECREF(key);
}
return py_obj;
}
static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list)
{
const uint32_t size = (*entry_data_list)->entry_data.data_size;
PyObject *py_obj = PyList_New(size);
if (NULL == py_obj) {
PyErr_NoMemory();
return NULL;
}
uint i;
// entry_data_list cannot start out NULL (see from_entry_data_list). We
// check it in the loop because it may become NULL.
// coverity[check_after_deref]
for (i = 0; i < size && entry_data_list; i++) {
*entry_data_list = (*entry_data_list)->next;
PyObject *value = from_entry_data_list(entry_data_list);
if (NULL == value) {
Py_DECREF(py_obj);
return NULL;
}
// PyList_SetItem 'steals' the reference
PyList_SetItem(py_obj, i, value);
}
return py_obj;
}
static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list)
{
uint64_t high = 0;
uint64_t low = 0;
#if MMDB_UINT128_IS_BYTE_ARRAY
int i;
for (i = 0; i < 8; i++) {
high = (high << 8) | entry_data_list->entry_data.uint128[i];
}
for (i = 8; i < 16; i++) {
low = (low << 8) | entry_data_list->entry_data.uint128[i];
}
#else
high = entry_data_list->entry_data.uint128 >> 64;
low = (uint64_t)entry_data_list->entry_data.uint128;
#endif
char *num_str = malloc(33);
if (NULL == num_str) {
PyErr_NoMemory();
return NULL;
}
snprintf(num_str, 33, "%016" PRIX64 "%016" PRIX64, high, low);
PyObject *py_obj = PyLong_FromString(num_str, NULL, 16);
free(num_str);
return py_obj;
}
static PyMethodDef Reader_methods[] = {
{ "get", Reader_get, METH_VARARGS,
"Get record for IP address" },
{ "metadata", Reader_metadata, METH_NOARGS,
"Returns metadata object for database" },
{ "close", Reader_close, METH_NOARGS, "Closes database"},
{ NULL, NULL, 0, NULL }
};
static PyTypeObject Reader_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_basicsize = sizeof(Reader_obj),
.tp_dealloc = Reader_dealloc,
.tp_doc = "Reader object",
.tp_flags = Py_TPFLAGS_DEFAULT,
.tp_methods = Reader_methods,
.tp_name = "Reader",
.tp_init = Reader_init,
};
static PyMethodDef Metadata_methods[] = {
{ NULL, NULL, 0, NULL }
};
/* *INDENT-OFF* */
static PyMemberDef Metadata_members[] = {
{ "binary_format_major_version", T_OBJECT, offsetof(
Metadata_obj, binary_format_major_version), READONLY, NULL },
{ "binary_format_minor_version", T_OBJECT, offsetof(
Metadata_obj, binary_format_minor_version), READONLY, NULL },
{ "build_epoch", T_OBJECT, offsetof(Metadata_obj, build_epoch),
READONLY, NULL },
{ "database_type", T_OBJECT, offsetof(Metadata_obj, database_type),
READONLY, NULL },
{ "description", T_OBJECT, offsetof(Metadata_obj, description),
READONLY, NULL },
{ "ip_version", T_OBJECT, offsetof(Metadata_obj, ip_version),
READONLY, NULL },
{ "languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY,
NULL },
{ "node_count", T_OBJECT, offsetof(Metadata_obj, node_count),
READONLY, NULL },
{ "record_size", T_OBJECT, offsetof(Metadata_obj, record_size),
READONLY, NULL },
{ NULL, 0, 0, 0, NULL }
};
/* *INDENT-ON* */
static PyTypeObject Metadata_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_basicsize = sizeof(Metadata_obj),
.tp_dealloc = Metadata_dealloc,
.tp_doc = "Metadata object",
.tp_flags = Py_TPFLAGS_DEFAULT,
.tp_members = Metadata_members,
.tp_methods = Metadata_methods,
.tp_name = "Metadata",
.tp_init = Metadata_init
};
static PyMethodDef MaxMindDB_methods[] = {
{ NULL, NULL, 0, NULL }
};
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef MaxMindDB_module = {
PyModuleDef_HEAD_INIT,
.m_name = "extension",
.m_doc = "This is a C extension to read MaxMind DB file format",
.m_methods = MaxMindDB_methods,
};
#endif
MOD_INIT(extension){
PyObject *m;
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&MaxMindDB_module);
#else
m = Py_InitModule("extension", MaxMindDB_methods);
#endif
if (!m) {
RETURN_MOD_INIT(NULL);
}
Reader_Type.tp_new = PyType_GenericNew;
if (PyType_Ready(&Reader_Type)) {
RETURN_MOD_INIT(NULL);
}
Py_INCREF(&Reader_Type);
PyModule_AddObject(m, "Reader", (PyObject *)&Reader_Type);
Metadata_Type.tp_new = PyType_GenericNew;
if (PyType_Ready(&Metadata_Type)) {
RETURN_MOD_INIT(NULL);
}
PyModule_AddObject(m, "extension", (PyObject *)&Metadata_Type);
PyObject* error_mod = PyImport_ImportModule("maxminddb.errors");
if (error_mod == NULL) {
RETURN_MOD_INIT(NULL);
}
MaxMindDB_error = PyObject_GetAttrString(error_mod, "InvalidDatabaseError");
Py_DECREF(error_mod);
if (MaxMindDB_error == NULL) {
RETURN_MOD_INIT(NULL);
}
Py_INCREF(MaxMindDB_error);
/* We primarily add it to the module for backwards compatibility */
PyModule_AddObject(m, "InvalidDatabaseError", MaxMindDB_error);
RETURN_MOD_INIT(m);
}

65
plugins/Sidebar/maxminddb/file.py
View file

@ -1,65 +0,0 @@
"""For internal use only. It provides a slice-like file reader."""
import os
try:
from multiprocessing import Lock
except ImportError:
from threading import Lock
class FileBuffer(object):
"""A slice-able file reader"""
def __init__(self, database):
self._handle = open(database, 'rb')
self._size = os.fstat(self._handle.fileno()).st_size
if not hasattr(os, 'pread'):
self._lock = Lock()
def __getitem__(self, key):
if isinstance(key, slice):
return self._read(key.stop - key.start, key.start)
elif isinstance(key, int):
return self._read(1, key)
else:
raise TypeError("Invalid argument type.")
def rfind(self, needle, start):
"""Reverse find needle from start"""
pos = self._read(self._size - start - 1, start).rfind(needle)
if pos == -1:
return pos
return start + pos
def size(self):
"""Size of file"""
return self._size
def close(self):
"""Close file"""
self._handle.close()
if hasattr(os, 'pread'):
def _read(self, buffersize, offset):
"""read that uses pread"""
# pylint: disable=no-member
return os.pread(self._handle.fileno(), buffersize, offset)
else:
def _read(self, buffersize, offset):
"""read with a lock
This lock is necessary as after a fork, the different processes
will share the same file table entry, even if we dup the fd, and
as such the same offsets. There does not appear to be a way to
duplicate the file table entry and we cannot re-open based on the
original path as that file may have replaced with another or
unlinked.
"""
with self._lock:
self._handle.seek(offset)
return self._handle.read(buffersize)

1897
plugins/Sidebar/maxminddb/ipaddr.py
View file

File diff suppressed because it is too large Load diff

221
plugins/Sidebar/maxminddb/reader.py
View file

@ -1,221 +0,0 @@
"""
maxminddb.reader
~~~~~~~~~~~~~~~~
This module contains the pure Python database reader and related classes.
"""
from __future__ import unicode_literals
try:
import mmap
except ImportError:
# pylint: disable=invalid-name
mmap = None
import struct
from maxminddb.compat import byte_from_int, int_from_byte, ipaddress
from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY
from maxminddb.decoder import Decoder
from maxminddb.errors import InvalidDatabaseError
from maxminddb.file import FileBuffer
class Reader(object):
"""
Instances of this class provide a reader for the MaxMind DB format. IP
addresses can be looked up using the ``get`` method.
"""
_DATA_SECTION_SEPARATOR_SIZE = 16
_METADATA_START_MARKER = b"\xAB\xCD\xEFMaxMind.com"
_ipv4_start = None
def __init__(self, database, mode=MODE_AUTO):
"""Reader for the MaxMind DB file format
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2
database file.
mode -- mode to open the database with. Valid mode are:
* MODE_MMAP - read from memory map.
* MODE_FILE - read database as standard file.
* MODE_MEMORY - load database into memory.
* MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default.
"""
if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP:
with open(database, 'rb') as db_file:
self._buffer = mmap.mmap(
db_file.fileno(), 0, access=mmap.ACCESS_READ)
self._buffer_size = self._buffer.size()
elif mode in (MODE_AUTO, MODE_FILE):
self._buffer = FileBuffer(database)
self._buffer_size = self._buffer.size()
elif mode == MODE_MEMORY:
with open(database, 'rb') as db_file:
self._buffer = db_file.read()
self._buffer_size = len(self._buffer)
else:
raise ValueError('Unsupported open mode ({0}). Only MODE_AUTO, '
' MODE_FILE, and MODE_MEMORY are support by the pure Python '
'Reader'.format(mode))
metadata_start = self._buffer.rfind(self._METADATA_START_MARKER,
max(0, self._buffer_size
- 128 * 1024))
if metadata_start == -1:
self.close()
raise InvalidDatabaseError('Error opening database file ({0}). '
'Is this a valid MaxMind DB file?'
''.format(database))
metadata_start += len(self._METADATA_START_MARKER)
metadata_decoder = Decoder(self._buffer, metadata_start)
(metadata, _) = metadata_decoder.decode(metadata_start)
self._metadata = Metadata(
**metadata) # pylint: disable=bad-option-value
self._decoder = Decoder(self._buffer, self._metadata.search_tree_size
+ self._DATA_SECTION_SEPARATOR_SIZE)
def metadata(self):
"""Return the metadata associated with the MaxMind DB file"""
return self._metadata
def get(self, ip_address):
"""Return the record for the ip_address in the MaxMind DB
Arguments:
ip_address -- an IP address in the standard string notation
"""
address = ipaddress.ip_address(ip_address)
if address.version == 6 and self._metadata.ip_version == 4:
raise ValueError('Error looking up {0}. You attempted to look up '
'an IPv6 address in an IPv4-only database.'.format(
ip_address))
pointer = self._find_address_in_tree(address)
return self._resolve_data_pointer(pointer) if pointer else None
def _find_address_in_tree(self, ip_address):
packed = ip_address.packed
bit_count = len(packed) * 8
node = self._start_node(bit_count)
for i in range(bit_count):
if node >= self._metadata.node_count:
break
bit = 1 & (int_from_byte(packed[i >> 3]) >> 7 - (i % 8))
node = self._read_node(node, bit)
if node == self._metadata.node_count:
# Record is empty
return 0
elif node > self._metadata.node_count:
return node
raise InvalidDatabaseError('Invalid node in search tree')
def _start_node(self, length):
if self._metadata.ip_version != 6 or length == 128:
return 0
# We are looking up an IPv4 address in an IPv6 tree. Skip over the
# first 96 nodes.
if self._ipv4_start:
return self._ipv4_start
node = 0
for _ in range(96):
if node >= self._metadata.node_count:
break
node = self._read_node(node, 0)
self._ipv4_start = node
return node
def _read_node(self, node_number, index):
base_offset = node_number * self._metadata.node_byte_size
record_size = self._metadata.record_size
if record_size == 24:
offset = base_offset + index * 3
node_bytes = b'\x00' + self._buffer[offset:offset + 3]
elif record_size == 28:
(middle,) = struct.unpack(
b'!B', self._buffer[base_offset + 3:base_offset + 4])
if index:
middle &= 0x0F
else:
middle = (0xF0 & middle) >> 4
offset = base_offset + index * 4
node_bytes = byte_from_int(
middle) + self._buffer[offset:offset + 3]
elif record_size == 32:
offset = base_offset + index * 4
node_bytes = self._buffer[offset:offset + 4]
else:
raise InvalidDatabaseError(
'Unknown record size: {0}'.format(record_size))
return struct.unpack(b'!I', node_bytes)[0]
def _resolve_data_pointer(self, pointer):
resolved = pointer - self._metadata.node_count + \
self._metadata.search_tree_size
if resolved > self._buffer_size:
raise InvalidDatabaseError(
"The MaxMind DB file's search tree is corrupt")
(data, _) = self._decoder.decode(resolved)
return data
def close(self):
"""Closes the MaxMind DB file and returns the resources to the system"""
# pylint: disable=unidiomatic-typecheck
if type(self._buffer) not in (str, bytes):
self._buffer.close()
class Metadata(object):
"""Metadata for the MaxMind DB reader"""
# pylint: disable=too-many-instance-attributes
def __init__(self, **kwargs):
"""Creates new Metadata object. kwargs are key/value pairs from spec"""
# Although I could just update __dict__, that is less obvious and it
# doesn't work well with static analysis tools and some IDEs
self.node_count = kwargs['node_count']
self.record_size = kwargs['record_size']
self.ip_version = kwargs['ip_version']
self.database_type = kwargs['database_type']
self.languages = kwargs['languages']
self.binary_format_major_version = kwargs[
'binary_format_major_version']
self.binary_format_minor_version = kwargs[
'binary_format_minor_version']
self.build_epoch = kwargs['build_epoch']
self.description = kwargs['description']
@property
def node_byte_size(self):
"""The size of a node in bytes"""
return self.record_size // 4
@property
def search_tree_size(self):
"""The size of the search tree"""
return self.node_count * self.node_byte_size
def __repr__(self):
args = ', '.join('%s=%r' % x for x in self.__dict__.items())
return '{module}.{class_name}({data})'.format(
module=self.__module__,
class_name=self.__class__.__name__,
data=args)