751 lines
24 KiB
Python
751 lines
24 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
logbook.queues
|
||
|
|
~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
This module implements queue backends.
|
||
|
|
|
||
|
|
:copyright: (c) 2010 by Armin Ronacher, Georg Brandl.
|
||
|
|
:license: BSD, see LICENSE for more details.
|
||
|
|
"""
|
||
|
|
import json
|
||
|
|
import threading
|
||
|
|
from threading import Thread, Lock
|
||
|
|
import platform
|
||
|
|
from logbook.base import NOTSET, LogRecord, dispatch_record
|
||
|
|
from logbook.handlers import Handler, WrapperHandler
|
||
|
|
from logbook.helpers import PY2, u
|
||
|
|
|
||
|
|
if PY2:
|
||
|
|
from Queue import Empty, Full, Queue as ThreadQueue
|
||
|
|
else:
|
||
|
|
from queue import Empty, Full, Queue as ThreadQueue
|
||
|
|
|
||
|
|
|
||
|
|
class RedisHandler(Handler):
|
||
|
|
"""A handler that sends log messages to a Redis instance.
|
||
|
|
|
||
|
|
It publishes each record as json dump. Requires redis module.
|
||
|
|
|
||
|
|
To receive such records you need to have a running instance of Redis.
|
||
|
|
|
||
|
|
Example setup::
|
||
|
|
|
||
|
|
handler = RedisHandler('http://127.0.0.1', port='9200', key='redis')
|
||
|
|
|
||
|
|
If your Redis instance is password protected, you can securely connect
|
||
|
|
passing your password when creating a RedisHandler object.
|
||
|
|
|
||
|
|
Example::
|
||
|
|
|
||
|
|
handler = RedisHandler(password='your_redis_password')
|
||
|
|
|
||
|
|
More info about the default buffer size: wp.me/p3tYJu-3b
|
||
|
|
"""
|
||
|
|
def __init__(self, host='127.0.0.1', port=6379, key='redis',
|
||
|
|
extra_fields=None, flush_threshold=128, flush_time=1,
|
||
|
|
level=NOTSET, filter=None, password=False, bubble=True,
|
||
|
|
context=None, push_method='rpush'):
|
||
|
|
Handler.__init__(self, level, filter, bubble)
|
||
|
|
try:
|
||
|
|
import redis
|
||
|
|
from redis import ResponseError
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('The redis library is required for '
|
||
|
|
'the RedisHandler')
|
||
|
|
|
||
|
|
self.redis = redis.Redis(host=host, port=port, password=password,
|
||
|
|
decode_responses=True)
|
||
|
|
try:
|
||
|
|
self.redis.ping()
|
||
|
|
except ResponseError:
|
||
|
|
raise ResponseError(
|
||
|
|
'The password provided is apparently incorrect')
|
||
|
|
self.key = key
|
||
|
|
self.extra_fields = extra_fields or {}
|
||
|
|
self.flush_threshold = flush_threshold
|
||
|
|
self.queue = []
|
||
|
|
self.lock = Lock()
|
||
|
|
self.push_method = push_method
|
||
|
|
|
||
|
|
# Set up a thread that flushes the queue every specified seconds
|
||
|
|
self._stop_event = threading.Event()
|
||
|
|
self._flushing_t = threading.Thread(target=self._flush_task,
|
||
|
|
args=(flush_time,
|
||
|
|
self._stop_event))
|
||
|
|
self._flushing_t.daemon = True
|
||
|
|
self._flushing_t.start()
|
||
|
|
|
||
|
|
def _flush_task(self, time, stop_event):
|
||
|
|
"""Calls the method _flush_buffer every certain time.
|
||
|
|
"""
|
||
|
|
while not self._stop_event.isSet():
|
||
|
|
with self.lock:
|
||
|
|
self._flush_buffer()
|
||
|
|
self._stop_event.wait(time)
|
||
|
|
|
||
|
|
def _flush_buffer(self):
|
||
|
|
"""Flushes the messaging queue into Redis.
|
||
|
|
|
||
|
|
All values are pushed at once for the same key.
|
||
|
|
|
||
|
|
The method rpush/lpush is defined by push_method argument
|
||
|
|
"""
|
||
|
|
if self.queue:
|
||
|
|
getattr(self.redis, self.push_method)(self.key, *self.queue)
|
||
|
|
self.queue = []
|
||
|
|
|
||
|
|
def disable_buffering(self):
|
||
|
|
"""Disables buffering.
|
||
|
|
|
||
|
|
If called, every single message will be directly pushed to Redis.
|
||
|
|
"""
|
||
|
|
self._stop_event.set()
|
||
|
|
self.flush_threshold = 1
|
||
|
|
|
||
|
|
def emit(self, record):
|
||
|
|
"""Emits a pair (key, value) to redis.
|
||
|
|
|
||
|
|
The key is the one provided when creating the handler, or redis if none
|
||
|
|
was provided. The value contains both the message and the hostname.
|
||
|
|
Extra values are also appended to the message.
|
||
|
|
"""
|
||
|
|
with self.lock:
|
||
|
|
r = {"message": record.msg,
|
||
|
|
"host": platform.node(),
|
||
|
|
"level": record.level_name,
|
||
|
|
"time": record.time.isoformat()}
|
||
|
|
r.update(self.extra_fields)
|
||
|
|
r.update(record.kwargs)
|
||
|
|
self.queue.append(json.dumps(r))
|
||
|
|
if len(self.queue) == self.flush_threshold:
|
||
|
|
self._flush_buffer()
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self._flush_buffer()
|
||
|
|
|
||
|
|
|
||
|
|
class MessageQueueHandler(Handler):
|
||
|
|
"""A handler that acts as a message queue publisher, which publishes each
|
||
|
|
record as json dump. Requires the kombu module.
|
||
|
|
|
||
|
|
The queue will be filled with JSON exported log records. To receive such
|
||
|
|
log records from a queue you can use the :class:`MessageQueueSubscriber`.
|
||
|
|
|
||
|
|
For an AMQP backend such as RabbitMQ::
|
||
|
|
|
||
|
|
handler = MessageQueueHandler('amqp://guest:guest@localhost//')
|
||
|
|
|
||
|
|
This requires the py-amqp or the librabbitmq client library.
|
||
|
|
|
||
|
|
For Redis (requires redis client library)::
|
||
|
|
|
||
|
|
handler = MessageQueueHandler('redis://localhost:8889/0')
|
||
|
|
|
||
|
|
For MongoDB (requires pymongo)::
|
||
|
|
|
||
|
|
handler = MessageQueueHandler('mongodb://localhost:27017/logging')
|
||
|
|
|
||
|
|
Several other backends are also supported.
|
||
|
|
Refer to the `kombu`_ documentation
|
||
|
|
|
||
|
|
.. _kombu: http://kombu.readthedocs.org/en/latest/introduction.html
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, uri=None, queue='logging', level=NOTSET,
|
||
|
|
filter=None, bubble=False):
|
||
|
|
Handler.__init__(self, level, filter, bubble)
|
||
|
|
try:
|
||
|
|
import kombu
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('The kombu library is required for '
|
||
|
|
'the RabbitMQSubscriber.')
|
||
|
|
if uri:
|
||
|
|
connection = kombu.Connection(uri)
|
||
|
|
|
||
|
|
self.queue = connection.SimpleQueue(queue)
|
||
|
|
|
||
|
|
def export_record(self, record):
|
||
|
|
"""Exports the record into a dictionary ready for JSON dumping.
|
||
|
|
"""
|
||
|
|
return record.to_dict(json_safe=True)
|
||
|
|
|
||
|
|
def emit(self, record):
|
||
|
|
self.queue.put(self.export_record(record))
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self.queue.close()
|
||
|
|
|
||
|
|
|
||
|
|
RabbitMQHandler = MessageQueueHandler
|
||
|
|
|
||
|
|
|
||
|
|
class ZeroMQHandler(Handler):
|
||
|
|
"""A handler that acts as a ZeroMQ publisher, which publishes each record
|
||
|
|
as json dump. Requires the pyzmq library.
|
||
|
|
|
||
|
|
The queue will be filled with JSON exported log records. To receive such
|
||
|
|
log records from a queue you can use the :class:`ZeroMQSubscriber`.
|
||
|
|
|
||
|
|
If `multi` is set to `True`, the handler will use a `PUSH` socket to
|
||
|
|
publish the records. This allows multiple handlers to use the same `uri`.
|
||
|
|
The records can be received by using the :class:`ZeroMQSubscriber` with
|
||
|
|
`multi` set to `True`.
|
||
|
|
|
||
|
|
|
||
|
|
Example setup::
|
||
|
|
|
||
|
|
handler = ZeroMQHandler('tcp://127.0.0.1:5000')
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, uri=None, level=NOTSET, filter=None, bubble=False,
|
||
|
|
context=None, multi=False):
|
||
|
|
Handler.__init__(self, level, filter, bubble)
|
||
|
|
try:
|
||
|
|
import zmq
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('The pyzmq library is required for '
|
||
|
|
'the ZeroMQHandler.')
|
||
|
|
#: the zero mq context
|
||
|
|
self.context = context or zmq.Context()
|
||
|
|
|
||
|
|
if multi:
|
||
|
|
#: the zero mq socket.
|
||
|
|
self.socket = self.context.socket(zmq.PUSH)
|
||
|
|
if uri is not None:
|
||
|
|
self.socket.connect(uri)
|
||
|
|
else:
|
||
|
|
#: the zero mq socket.
|
||
|
|
self.socket = self.context.socket(zmq.PUB)
|
||
|
|
if uri is not None:
|
||
|
|
self.socket.bind(uri)
|
||
|
|
|
||
|
|
def export_record(self, record):
|
||
|
|
"""Exports the record into a dictionary ready for JSON dumping."""
|
||
|
|
return record.to_dict(json_safe=True)
|
||
|
|
|
||
|
|
def emit(self, record):
|
||
|
|
self.socket.send(json.dumps(
|
||
|
|
self.export_record(record)).encode("utf-8"))
|
||
|
|
|
||
|
|
def close(self, linger=-1):
|
||
|
|
self.socket.close(linger)
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
# When the Handler is deleted we must close our socket in a
|
||
|
|
# non-blocking fashion (using linger).
|
||
|
|
# Otherwise it can block indefinitely, for example if the Subscriber is
|
||
|
|
# not reachable.
|
||
|
|
# If messages are pending on the socket, we wait 100ms for them to be
|
||
|
|
# sent then we discard them.
|
||
|
|
self.close(linger=100)
|
||
|
|
|
||
|
|
|
||
|
|
class ThreadController(object):
|
||
|
|
"""A helper class used by queue subscribers to control the background
|
||
|
|
thread. This is usually created and started in one go by
|
||
|
|
:meth:`~logbook.queues.ZeroMQSubscriber.dispatch_in_background` or
|
||
|
|
a comparable function.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, subscriber, setup=None):
|
||
|
|
self.setup = setup
|
||
|
|
self.subscriber = subscriber
|
||
|
|
self.running = False
|
||
|
|
self._thread = None
|
||
|
|
|
||
|
|
def start(self):
|
||
|
|
"""Starts the task thread."""
|
||
|
|
self.running = True
|
||
|
|
self._thread = Thread(target=self._target)
|
||
|
|
self._thread.setDaemon(True)
|
||
|
|
self._thread.start()
|
||
|
|
|
||
|
|
def stop(self):
|
||
|
|
"""Stops the task thread."""
|
||
|
|
if self.running:
|
||
|
|
self.running = False
|
||
|
|
self._thread.join()
|
||
|
|
self._thread = None
|
||
|
|
|
||
|
|
def _target(self):
|
||
|
|
if self.setup is not None:
|
||
|
|
self.setup.push_thread()
|
||
|
|
try:
|
||
|
|
while self.running:
|
||
|
|
self.subscriber.dispatch_once(timeout=0.05)
|
||
|
|
finally:
|
||
|
|
if self.setup is not None:
|
||
|
|
self.setup.pop_thread()
|
||
|
|
|
||
|
|
|
||
|
|
class SubscriberBase(object):
|
||
|
|
"""Baseclass for all subscribers."""
|
||
|
|
|
||
|
|
def recv(self, timeout=None):
|
||
|
|
"""Receives a single record from the socket. Timeout of 0 means
|
||
|
|
nonblocking, `None` means blocking and otherwise it's a timeout in
|
||
|
|
seconds after which the function just returns with `None`.
|
||
|
|
|
||
|
|
Subclasses have to override this.
|
||
|
|
"""
|
||
|
|
raise NotImplementedError()
|
||
|
|
|
||
|
|
def dispatch_once(self, timeout=None):
|
||
|
|
"""Receives one record from the socket, loads it and dispatches it. Returns
|
||
|
|
`True` if something was dispatched or `False` if it timed out.
|
||
|
|
"""
|
||
|
|
rv = self.recv(timeout)
|
||
|
|
if rv is not None:
|
||
|
|
dispatch_record(rv)
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def dispatch_forever(self):
|
||
|
|
"""Starts a loop that dispatches log records forever."""
|
||
|
|
while 1:
|
||
|
|
self.dispatch_once()
|
||
|
|
|
||
|
|
def dispatch_in_background(self, setup=None):
|
||
|
|
"""Starts a new daemonized thread that dispatches in the background.
|
||
|
|
An optional handler setup can be provided that pushed to the new
|
||
|
|
thread (can be any :class:`logbook.base.StackedObject`).
|
||
|
|
|
||
|
|
Returns a :class:`ThreadController` object for shutting down
|
||
|
|
the background thread. The background thread will already be
|
||
|
|
running when this function returns.
|
||
|
|
"""
|
||
|
|
controller = ThreadController(self, setup)
|
||
|
|
controller.start()
|
||
|
|
return controller
|
||
|
|
|
||
|
|
|
||
|
|
class MessageQueueSubscriber(SubscriberBase):
|
||
|
|
"""A helper that acts as a message queue subscriber and will dispatch
|
||
|
|
received log records to the active handler setup. There are multiple ways
|
||
|
|
to use this class.
|
||
|
|
|
||
|
|
It can be used to receive log records from a queue::
|
||
|
|
|
||
|
|
subscriber = MessageQueueSubscriber('mongodb://localhost:27017/logging')
|
||
|
|
record = subscriber.recv()
|
||
|
|
|
||
|
|
But it can also be used to receive and dispatch these in one go::
|
||
|
|
|
||
|
|
with target_handler:
|
||
|
|
subscriber = MessageQueueSubscriber('mongodb://localhost:27017/logging')
|
||
|
|
subscriber.dispatch_forever()
|
||
|
|
|
||
|
|
This will take all the log records from that queue and dispatch them
|
||
|
|
over to `target_handler`. If you want you can also do that in the
|
||
|
|
background::
|
||
|
|
|
||
|
|
subscriber = MessageQueueSubscriber('mongodb://localhost:27017/logging')
|
||
|
|
controller = subscriber.dispatch_in_background(target_handler)
|
||
|
|
|
||
|
|
The controller returned can be used to shut down the background
|
||
|
|
thread::
|
||
|
|
|
||
|
|
controller.stop()
|
||
|
|
"""
|
||
|
|
def __init__(self, uri=None, queue='logging'):
|
||
|
|
try:
|
||
|
|
import kombu
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('The kombu library is required.')
|
||
|
|
if uri:
|
||
|
|
connection = kombu.Connection(uri)
|
||
|
|
|
||
|
|
self.queue = connection.SimpleQueue(queue)
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
try:
|
||
|
|
self.close()
|
||
|
|
except AttributeError:
|
||
|
|
# subscriber partially created
|
||
|
|
pass
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self.queue.close()
|
||
|
|
|
||
|
|
def recv(self, timeout=None):
|
||
|
|
"""Receives a single record from the socket. Timeout of 0 means
|
||
|
|
nonblocking, `None` means blocking and otherwise it's a timeout in
|
||
|
|
seconds after which the function just returns with `None`.
|
||
|
|
"""
|
||
|
|
if timeout == 0:
|
||
|
|
try:
|
||
|
|
rv = self.queue.get(block=False)
|
||
|
|
except Exception:
|
||
|
|
return
|
||
|
|
else:
|
||
|
|
rv = self.queue.get(timeout=timeout)
|
||
|
|
|
||
|
|
log_record = rv.payload
|
||
|
|
rv.ack()
|
||
|
|
|
||
|
|
return LogRecord.from_dict(log_record)
|
||
|
|
|
||
|
|
|
||
|
|
RabbitMQSubscriber = MessageQueueSubscriber
|
||
|
|
|
||
|
|
|
||
|
|
class ZeroMQSubscriber(SubscriberBase):
|
||
|
|
"""A helper that acts as ZeroMQ subscriber and will dispatch received
|
||
|
|
log records to the active handler setup. There are multiple ways to
|
||
|
|
use this class.
|
||
|
|
|
||
|
|
It can be used to receive log records from a queue::
|
||
|
|
|
||
|
|
subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000')
|
||
|
|
record = subscriber.recv()
|
||
|
|
|
||
|
|
But it can also be used to receive and dispatch these in one go::
|
||
|
|
|
||
|
|
with target_handler:
|
||
|
|
subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000')
|
||
|
|
subscriber.dispatch_forever()
|
||
|
|
|
||
|
|
This will take all the log records from that queue and dispatch them
|
||
|
|
over to `target_handler`. If you want you can also do that in the
|
||
|
|
background::
|
||
|
|
|
||
|
|
subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000')
|
||
|
|
controller = subscriber.dispatch_in_background(target_handler)
|
||
|
|
|
||
|
|
The controller returned can be used to shut down the background
|
||
|
|
thread::
|
||
|
|
|
||
|
|
controller.stop()
|
||
|
|
|
||
|
|
If `multi` is set to `True`, the subscriber will use a `PULL` socket
|
||
|
|
and listen to records published by a `PUSH` socket (usually via a
|
||
|
|
:class:`ZeroMQHandler` with `multi` set to `True`). This allows a
|
||
|
|
single subscriber to dispatch multiple handlers.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, uri=None, context=None, multi=False):
|
||
|
|
try:
|
||
|
|
import zmq
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('The pyzmq library is required for '
|
||
|
|
'the ZeroMQSubscriber.')
|
||
|
|
self._zmq = zmq
|
||
|
|
|
||
|
|
#: the zero mq context
|
||
|
|
self.context = context or zmq.Context()
|
||
|
|
|
||
|
|
if multi:
|
||
|
|
#: the zero mq socket.
|
||
|
|
self.socket = self.context.socket(zmq.PULL)
|
||
|
|
if uri is not None:
|
||
|
|
self.socket.bind(uri)
|
||
|
|
else:
|
||
|
|
#: the zero mq socket.
|
||
|
|
self.socket = self.context.socket(zmq.SUB)
|
||
|
|
if uri is not None:
|
||
|
|
self.socket.connect(uri)
|
||
|
|
self.socket.setsockopt_unicode(zmq.SUBSCRIBE, u(''))
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
try:
|
||
|
|
self.close()
|
||
|
|
except AttributeError:
|
||
|
|
# subscriber partially created
|
||
|
|
pass
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
"""Closes the zero mq socket."""
|
||
|
|
self.socket.close()
|
||
|
|
|
||
|
|
def recv(self, timeout=None):
|
||
|
|
"""Receives a single record from the socket. Timeout of 0 means
|
||
|
|
nonblocking, `None` means blocking and otherwise it's a timeout in
|
||
|
|
seconds after which the function just returns with `None`.
|
||
|
|
"""
|
||
|
|
if timeout is None:
|
||
|
|
rv = self.socket.recv()
|
||
|
|
elif not timeout:
|
||
|
|
rv = self.socket.recv(self._zmq.NOBLOCK)
|
||
|
|
if rv is None:
|
||
|
|
return
|
||
|
|
else:
|
||
|
|
if not self._zmq.select([self.socket], [], [], timeout)[0]:
|
||
|
|
return
|
||
|
|
rv = self.socket.recv(self._zmq.NOBLOCK)
|
||
|
|
if not PY2:
|
||
|
|
rv = rv.decode("utf-8")
|
||
|
|
return LogRecord.from_dict(json.loads(rv))
|
||
|
|
|
||
|
|
|
||
|
|
def _fix_261_mplog():
|
||
|
|
"""necessary for older python's to disable a broken monkeypatch
|
||
|
|
in the logging module. See multiprocessing/util.py for the
|
||
|
|
hasattr() check. At least in Python 2.6.1 the multiprocessing
|
||
|
|
module is not imported by logging and as such the test in
|
||
|
|
the util fails.
|
||
|
|
"""
|
||
|
|
import logging
|
||
|
|
import multiprocessing
|
||
|
|
logging.multiprocessing = multiprocessing
|
||
|
|
|
||
|
|
|
||
|
|
class MultiProcessingHandler(Handler):
|
||
|
|
"""Implements a handler that dispatches over a queue to a different
|
||
|
|
process. It is connected to a subscriber with a
|
||
|
|
:class:`multiprocessing.Queue`::
|
||
|
|
|
||
|
|
from multiprocessing import Queue
|
||
|
|
from logbook.queues import MultiProcessingHandler
|
||
|
|
queue = Queue(-1)
|
||
|
|
handler = MultiProcessingHandler(queue)
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, queue, level=NOTSET, filter=None, bubble=False):
|
||
|
|
Handler.__init__(self, level, filter, bubble)
|
||
|
|
self.queue = queue
|
||
|
|
_fix_261_mplog()
|
||
|
|
|
||
|
|
def emit(self, record):
|
||
|
|
self.queue.put_nowait(record.to_dict(json_safe=True))
|
||
|
|
|
||
|
|
|
||
|
|
class MultiProcessingSubscriber(SubscriberBase):
|
||
|
|
"""Receives log records from the given multiprocessing queue and
|
||
|
|
dispatches them to the active handler setup. Make sure to use the same
|
||
|
|
queue for both handler and subscriber. Idaelly the queue is set
|
||
|
|
up with maximum size (``-1``)::
|
||
|
|
|
||
|
|
from multiprocessing import Queue
|
||
|
|
queue = Queue(-1)
|
||
|
|
|
||
|
|
It can be used to receive log records from a queue::
|
||
|
|
|
||
|
|
subscriber = MultiProcessingSubscriber(queue)
|
||
|
|
record = subscriber.recv()
|
||
|
|
|
||
|
|
But it can also be used to receive and dispatch these in one go::
|
||
|
|
|
||
|
|
with target_handler:
|
||
|
|
subscriber = MultiProcessingSubscriber(queue)
|
||
|
|
subscriber.dispatch_forever()
|
||
|
|
|
||
|
|
This will take all the log records from that queue and dispatch them
|
||
|
|
over to `target_handler`. If you want you can also do that in the
|
||
|
|
background::
|
||
|
|
|
||
|
|
subscriber = MultiProcessingSubscriber(queue)
|
||
|
|
controller = subscriber.dispatch_in_background(target_handler)
|
||
|
|
|
||
|
|
The controller returned can be used to shut down the background
|
||
|
|
thread::
|
||
|
|
|
||
|
|
controller.stop()
|
||
|
|
|
||
|
|
If no queue is provided the subscriber will create one. This one can the
|
||
|
|
be used by handlers::
|
||
|
|
|
||
|
|
subscriber = MultiProcessingSubscriber()
|
||
|
|
handler = MultiProcessingHandler(subscriber.queue)
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, queue=None):
|
||
|
|
if queue is None:
|
||
|
|
from multiprocessing import Queue
|
||
|
|
queue = Queue(-1)
|
||
|
|
self.queue = queue
|
||
|
|
_fix_261_mplog()
|
||
|
|
|
||
|
|
def recv(self, timeout=None):
|
||
|
|
if timeout is None:
|
||
|
|
rv = self.queue.get()
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
rv = self.queue.get(block=True, timeout=timeout)
|
||
|
|
except Empty:
|
||
|
|
return None
|
||
|
|
return LogRecord.from_dict(rv)
|
||
|
|
|
||
|
|
|
||
|
|
class ExecnetChannelHandler(Handler):
|
||
|
|
"""Implements a handler that dispatches over a execnet channel
|
||
|
|
to a different process.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, channel, level=NOTSET, filter=None, bubble=False):
|
||
|
|
Handler.__init__(self, level, filter, bubble)
|
||
|
|
self.channel = channel
|
||
|
|
|
||
|
|
def emit(self, record):
|
||
|
|
self.channel.send(record.to_dict(json_safe=True))
|
||
|
|
|
||
|
|
|
||
|
|
class ExecnetChannelSubscriber(SubscriberBase):
|
||
|
|
"""subscribes to a execnet channel"""
|
||
|
|
|
||
|
|
def __init__(self, channel):
|
||
|
|
self.channel = channel
|
||
|
|
|
||
|
|
def recv(self, timeout=None):
|
||
|
|
try:
|
||
|
|
rv = self.channel.receive(timeout=timeout)
|
||
|
|
except self.channel.RemoteError:
|
||
|
|
# XXX: handle
|
||
|
|
return None
|
||
|
|
except (self.channel.TimeoutError, EOFError):
|
||
|
|
return None
|
||
|
|
else:
|
||
|
|
return LogRecord.from_dict(rv)
|
||
|
|
|
||
|
|
|
||
|
|
class TWHThreadController(object):
|
||
|
|
"""A very basic thread controller that pulls things in from a
|
||
|
|
queue and sends it to a handler. Both queue and handler are
|
||
|
|
taken from the passed :class:`ThreadedWrapperHandler`.
|
||
|
|
"""
|
||
|
|
class Command(object):
|
||
|
|
stop = object()
|
||
|
|
emit = object()
|
||
|
|
emit_batch = object()
|
||
|
|
|
||
|
|
def __init__(self, wrapper_handler):
|
||
|
|
self.wrapper_handler = wrapper_handler
|
||
|
|
self.running = False
|
||
|
|
self._thread = None
|
||
|
|
|
||
|
|
def start(self):
|
||
|
|
"""Starts the task thread."""
|
||
|
|
self.running = True
|
||
|
|
self._thread = Thread(target=self._target)
|
||
|
|
self._thread.setDaemon(True)
|
||
|
|
self._thread.start()
|
||
|
|
|
||
|
|
def stop(self):
|
||
|
|
"""Stops the task thread."""
|
||
|
|
if self.running:
|
||
|
|
self.wrapper_handler.queue.put_nowait((self.Command.stop, ))
|
||
|
|
self._thread.join()
|
||
|
|
self._thread = None
|
||
|
|
|
||
|
|
def _target(self):
|
||
|
|
while 1:
|
||
|
|
item = self.wrapper_handler.queue.get()
|
||
|
|
command, data = item[0], item[1:]
|
||
|
|
if command is self.Command.stop:
|
||
|
|
self.running = False
|
||
|
|
break
|
||
|
|
elif command is self.Command.emit:
|
||
|
|
(record, ) = data
|
||
|
|
self.wrapper_handler.handler.emit(record)
|
||
|
|
elif command is self.Command.emit_batch:
|
||
|
|
record, reason = data
|
||
|
|
self.wrapper_handler.handler.emit_batch(record, reason)
|
||
|
|
|
||
|
|
|
||
|
|
class ThreadedWrapperHandler(WrapperHandler):
|
||
|
|
"""This handled uses a single background thread to dispatch log records
|
||
|
|
to a specific other handler using an internal queue. The idea is that if
|
||
|
|
you are using a handler that requires some time to hand off the log records
|
||
|
|
(such as the mail handler) and would block your request, you can let
|
||
|
|
Logbook do that in a background thread.
|
||
|
|
|
||
|
|
The threaded wrapper handler will automatically adopt the methods and
|
||
|
|
properties of the wrapped handler. All the values will be reflected:
|
||
|
|
|
||
|
|
>>> twh = ThreadedWrapperHandler(TestHandler())
|
||
|
|
>>> from logbook import WARNING
|
||
|
|
>>> twh.level_name = 'WARNING'
|
||
|
|
>>> twh.handler.level_name
|
||
|
|
'WARNING'
|
||
|
|
"""
|
||
|
|
_direct_attrs = frozenset(['handler', 'queue', 'controller'])
|
||
|
|
|
||
|
|
def __init__(self, handler, maxsize=0):
|
||
|
|
WrapperHandler.__init__(self, handler)
|
||
|
|
self.queue = ThreadQueue(maxsize)
|
||
|
|
self.controller = TWHThreadController(self)
|
||
|
|
self.controller.start()
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self.controller.stop()
|
||
|
|
self.handler.close()
|
||
|
|
|
||
|
|
def emit(self, record):
|
||
|
|
item = (TWHThreadController.Command.emit, record)
|
||
|
|
try:
|
||
|
|
self.queue.put_nowait(item)
|
||
|
|
except Full:
|
||
|
|
# silently drop
|
||
|
|
pass
|
||
|
|
|
||
|
|
def emit_batch(self, records, reason):
|
||
|
|
item = (TWHThreadController.Command.emit_batch, records, reason)
|
||
|
|
try:
|
||
|
|
self.queue.put_nowait(item)
|
||
|
|
except Full:
|
||
|
|
# silently drop
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
class GroupMember(ThreadController):
|
||
|
|
def __init__(self, subscriber, queue):
|
||
|
|
ThreadController.__init__(self, subscriber, None)
|
||
|
|
self.queue = queue
|
||
|
|
|
||
|
|
def _target(self):
|
||
|
|
if self.setup is not None:
|
||
|
|
self.setup.push_thread()
|
||
|
|
try:
|
||
|
|
while self.running:
|
||
|
|
record = self.subscriber.recv()
|
||
|
|
if record:
|
||
|
|
try:
|
||
|
|
self.queue.put(record, timeout=0.05)
|
||
|
|
except Full:
|
||
|
|
pass
|
||
|
|
finally:
|
||
|
|
if self.setup is not None:
|
||
|
|
self.setup.pop_thread()
|
||
|
|
|
||
|
|
|
||
|
|
class SubscriberGroup(SubscriberBase):
|
||
|
|
"""This is a subscriber which represents a group of subscribers.
|
||
|
|
|
||
|
|
This is helpful if you are writing a server-like application which has
|
||
|
|
"slaves". This way a user is easily able to view every log record which
|
||
|
|
happened somewhere in the entire system without having to check every
|
||
|
|
single slave::
|
||
|
|
|
||
|
|
subscribers = SubscriberGroup([
|
||
|
|
MultiProcessingSubscriber(queue),
|
||
|
|
ZeroMQSubscriber('tcp://127.0.0.1:5000')
|
||
|
|
])
|
||
|
|
with target_handler:
|
||
|
|
subscribers.dispatch_forever()
|
||
|
|
"""
|
||
|
|
def __init__(self, subscribers=None, queue_limit=10):
|
||
|
|
self.members = []
|
||
|
|
self.queue = ThreadQueue(queue_limit)
|
||
|
|
for subscriber in subscribers or []:
|
||
|
|
self.add(subscriber)
|
||
|
|
|
||
|
|
def add(self, subscriber):
|
||
|
|
"""Adds the given `subscriber` to the group."""
|
||
|
|
member = GroupMember(subscriber, self.queue)
|
||
|
|
member.start()
|
||
|
|
self.members.append(member)
|
||
|
|
|
||
|
|
def recv(self, timeout=None):
|
||
|
|
try:
|
||
|
|
return self.queue.get(timeout=timeout)
|
||
|
|
except Empty:
|
||
|
|
return
|
||
|
|
|
||
|
|
def stop(self):
|
||
|
|
"""Stops the group from internally recieving any more messages, once the
|
||
|
|
internal queue is exhausted :meth:`recv` will always return `None`.
|
||
|
|
"""
|
||
|
|
for member in self.members:
|
||
|
|
self.member.stop()
|