diff --git a/README.rst b/README.rst index aec1c3a..b3f79f3 100644 --- a/README.rst +++ b/README.rst @@ -41,10 +41,13 @@ Using pip:: pip install python-logstash + Usage ===== -``LogstashHandler`` is a custom logging handler which sends Logstash messages using UDP. +``UDPLogstashHandler`` is a custom logging handler which sends JSON formatted +Logstash messages using UDP. ``TCPLogstashHandler`` provides the same +functionality via TCP. For example:: @@ -56,14 +59,24 @@ For example:: test_logger = logging.getLogger('python-logstash-logger') test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1)) + + test_logger.addHandler( + logstash.UDPLogstashHandler( + host, + 5959, + version=1, + default_fields=('levelname', 'name', 'lineno', 'funcName'), + ) + ) + + # alternatively use the TCP handler # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1)) test_logger.error('python-logstash: test logstash error message.') test_logger.info('python-logstash: test logstash info message.') test_logger.warning('python-logstash: test logstash warning message.') - # add extra field to logstash message + # add extra fields to logstash message extra = { 'test_string': 'python version: ' + repr(sys.version_info), 'test_boolean': True, @@ -74,12 +87,15 @@ For example:: } test_logger.info('python-logstash: test extra fields', extra=extra) -When using ``extra`` field make sure you don't use reserved names. From `Python documentation `_. +When using ``extra`` field make sure you don't use reserved names. +From `Python documentation `_. + | "The keys in the dictionary passed in extra should not clash with the keys used by the logging system. (See the `Formatter `_ documentation for more information on which keys are used by the logging system.)" -To use the AMQPLogstashHandler you will need to install pika first. - pip install pika +To use the ``AMQPLogstashHandler`` you will need to install pika first. + + pip install pika For example:: @@ -88,20 +104,26 @@ For example:: test_logger = logging.getLogger('python-logstash-logger') test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1)) + test_logger.addHandler( + logstash.AMQPLogstashHandler( + host='localhost', + version=1, + default_fields=('levelname', 'name', 'lineno', 'funcName') + ) + ) test_logger.info('python-logstash: test logstash info message.') try: 1/0 except: test_logger.exception('python-logstash-logger: Exception with stack trace!') + - - -Using with Django +Usage with Django ================= -Modify your ``settings.py`` to integrate ``python-logstash`` with Django's logging:: +Modify your ``settings.py`` to integrate ``python-logstash`` with Django's +logging:: LOGGING = { ... @@ -110,11 +132,23 @@ Modify your ``settings.py`` to integrate ``python-logstash`` with Django's loggi 'level': 'DEBUG', 'class': 'logstash.LogstashHandler', 'host': 'localhost', - 'port': 5959, # Default value: 5959 - 'version': 1, # Version of logstash event schema. Default value: 0 (for backward compatibility of the library) - 'message_type': 'logstash', # 'type' field in logstash message. Default value: 'logstash'. - 'fqdn': False, # Fully qualified domain name. Default value: false. - 'tags': ['tag1', 'tag2'], # list of tags. Default: None. + # default: 5959 + 'port': 5959, + # Version of logstash event schema, default: 0 (for backward compatibility of the library) + 'version': 1, + # 'type' field in logstash message, default: 'logstash' + 'message_type': 'logstash', + # Fully qualified domain name, default: False + 'fqdn': False, + # list of tags, default: None + 'tags': ['tag1', 'tag2'], + # log record attributes to include in the message, default: ('levelname', 'name') + 'default_fields': ( + 'levelname', + 'name', + 'lineno', + 'funcName', + ) }, }, 'loggers': { diff --git a/logstash/formatter.py b/logstash/formatter.py index f5c4424..0008876 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -3,24 +3,52 @@ import socket import sys from datetime import datetime + try: import json except ImportError: import simplejson as json -class LogstashFormatterBase(logging.Formatter): +# a mapping of attribute names on log record to output keys +# contains only those that need a mapping, fallback is record attribute name +field_map = { + 'name': 'logger', +} - def __init__(self, message_type='Logstash', tags=None, fqdn=False): + +class LogstashFormatterBase(logging.Formatter): + def __init__(self, message_type='Logstash', tags=None, fqdn=False, + default_fields=None, exc_fields=None): self.message_type = message_type self.tags = tags if tags is not None else [] + self.default_fields = default_fields \ + if default_fields is not None \ + else ( + 'levelname', + 'name', + ) + self.exc_fields = exc_fields \ + if exc_fields is not None \ + else ( + 'exc_info', + # funcName was added in 2.5 + 'funcName', + 'lineno', + 'process', + # processName was added in 2.6 + 'processName', + 'threadName', + ) + if fqdn: self.host = socket.getfqdn() else: self.host = socket.gethostname() - def get_extra_fields(self, record): + @staticmethod + def get_extra_fields(record): # The list contains all the attributes listed in # http://docs.python.org/library/logging.html#logrecord-attributes skip_list = ( @@ -46,23 +74,38 @@ def get_extra_fields(self, record): return fields - def get_debug_fields(self, record): - fields = { - 'stack_trace': self.format_exception(record.exc_info), - 'lineno': record.lineno, - 'process': record.process, - 'thread_name': record.threadName, - } - - # funcName was added in 2.5 - if not getattr(record, 'funcName', None): - fields['funcName'] = record.funcName - - # processName was added in 2.6 - if not getattr(record, 'processName', None): - fields['processName'] = record.processName - - return fields + @staticmethod + def get_fields(record, field_names): + """ + Get a dict with key/value pairs for all fields in `field_names` from + the `record`. Keys are translated according to the `field_map` and + special values formatted using `format_field()`. + + :param record: log record + :param field_names: list of record attribute names + :return: dict, ready for output + """ + return dict([ + ( + field_map.get(record_key, record_key), + self.format_field(record_key, getattr(record, record_key, None)) + ) + for record_key + in field_names + ]) + + def format_field(self, record_key, value): + """ + Apply special formatting to certain record fields. + + :param record_key: record attribute name + :param value: attribute value to format + :return: the formatted value or original value + """ + if record_key == 'exc_info': + return self.format_exception(record.exc_info) + + return value @classmethod def format_source(cls, message_type, host, path): @@ -71,7 +114,8 @@ def format_source(cls, message_type, host, path): @classmethod def format_timestamp(cls, time): tstamp = datetime.utcfromtimestamp(time) - return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (tstamp.microsecond / 1000) + "Z" + return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % \ + (tstamp.microsecond / 1000) + "Z" @classmethod def format_exception(cls, exc_info): @@ -84,6 +128,7 @@ def serialize(cls, message): else: return bytes(json.dumps(message), 'utf-8') + class LogstashFormatterVersion0(LogstashFormatterBase): version = 0 @@ -93,7 +138,7 @@ def format(self, record): '@timestamp': self.format_timestamp(record.created), '@message': record.getMessage(), '@source': self.format_source(self.message_type, self.host, - record.pathname), + record.pathname), '@source_host': self.host, '@source_path': record.pathname, '@tags': self.tags, @@ -104,18 +149,20 @@ def format(self, record): }, } + # Add default extra fields + message['@fields'].update(self.get_fields(record, self.default_fields)) + # Add extra fields message['@fields'].update(self.get_extra_fields(record)) # If exception, add debug info if record.exc_info: - message['@fields'].update(self.get_debug_fields(record)) + message['@fields'].update(self.get_fields(record, self.exc_fields)) return self.serialize(message) class LogstashFormatterVersion1(LogstashFormatterBase): - def format(self, record): # Create message dict message = { @@ -132,11 +179,14 @@ def format(self, record): 'logger_name': record.name, } + # Add default extra fields + message.update(self.get_fields(record, self.default_fields)) + # Add extra fields message.update(self.get_extra_fields(record)) # If exception, add debug info if record.exc_info: - message.update(self.get_debug_fields(record)) + message.update(self.get_fields(record, self.exc_fields)) return self.serialize(message) diff --git a/logstash/handler_amqp.py b/logstash/handler_amqp.py index 4209964..c5c1670 100644 --- a/logstash/handler_amqp.py +++ b/logstash/handler_amqp.py @@ -47,7 +47,7 @@ def __init__(self, host='localhost', port=5672, username='guest', password='guest', exchange='logstash', exchange_type='fanout', virtual_host='/', message_type='logstash', tags=None, durable=False, passive=False, version=0, extra_fields=True, - fqdn=False, facility=None, exchange_routing_key=''): + fqdn=False, facility=None, exchange_routing_key='', **kwargs): # AMQP parameters @@ -68,7 +68,7 @@ def __init__(self, host='localhost', port=5672, username='guest', self.tags = tags or [] fn = formatter.LogstashFormatterVersion1 if version == 1 \ else formatter.LogstashFormatterVersion0 - self.formatter = fn(message_type, tags, fqdn) + self.formatter = fn(message_type, tags, fqdn, **kwargs) # Standard logging parameters self.extra_fields = extra_fields diff --git a/logstash/handler_tcp.py b/logstash/handler_tcp.py index cc687bd..07ce8e7 100644 --- a/logstash/handler_tcp.py +++ b/logstash/handler_tcp.py @@ -14,12 +14,12 @@ class TCPLogstashHandler(SocketHandler, object): :param tags: list of tags for a logger (default is None). """ - def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0): + def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0, **kwargs): super(TCPLogstashHandler, self).__init__(host, port) if version == 1: - self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn) + self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn, **kwargs) else: - self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn) + self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn, **kwargs) def makePickle(self, record): return self.formatter.format(record) + b'\n'