From f019cbe4d577461c0666b67106bc9572140945a7 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:50:59 +0200 Subject: [PATCH 1/9] PEP8 fixes --- logstash/formatter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/logstash/formatter.py b/logstash/formatter.py index edbe7ca..f9dbbe5 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -3,6 +3,7 @@ import socket import sys from datetime import datetime + try: import json except ImportError: @@ -70,7 +71,8 @@ def format_source(cls, message_type, host, path): @classmethod def format_timestamp(cls, time): tstamp = datetime.utcfromtimestamp(time) - return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (tstamp.microsecond / 1000) + "Z" + return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % \ + (tstamp.microsecond / 1000) + "Z" @classmethod def format_exception(cls, exc_info): @@ -83,6 +85,7 @@ def serialize(cls, message): else: return bytes(json.dumps(message), 'utf-8') + class LogstashFormatterVersion0(LogstashFormatterBase): version = 0 @@ -92,7 +95,7 @@ def format(self, record): '@timestamp': self.format_timestamp(record.created), '@message': record.getMessage(), '@source': self.format_source(self.message_type, self.host, - record.pathname), + record.pathname), '@source_host': self.host, '@source_path': record.pathname, '@tags': self.tags, From 62885d03e8a44c4c6db65044c57407be02f53013 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:51:59 +0200 Subject: [PATCH 2/9] Add default_fields and exc_fields parameters --- logstash/formatter.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/logstash/formatter.py b/logstash/formatter.py index f9dbbe5..9124750 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -10,12 +10,31 @@ import simplejson as json -class LogstashFormatterBase(logging.Formatter): - def __init__(self, message_type='Logstash', tags=None, fqdn=False): + +class LogstashFormatterBase(logging.Formatter): + def __init__(self, message_type='Logstash', tags=None, fqdn=False, + default_fields=None, exc_fields=None): self.message_type = message_type self.tags = tags if tags is not None else [] + self.default_fields = default_fields \ + if default_fields is not None \ + else ( + 'levelname', + 'name', + ) + self.exc_fields = exc_fields \ + if exc_fields is not None \ + else ( + 'exc_info', + 'funcName', + 'lineno', + 'process', + 'processName', + 'threadName', + ) + if fqdn: self.host = socket.getfqdn() else: From ffb6179c00884cc6be3bdf4b4364b80f3e16f05d Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:52:12 +0200 Subject: [PATCH 3/9] Pass on kwargs from handler to formatter --- logstash/handler_tcp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logstash/handler_tcp.py b/logstash/handler_tcp.py index cc687bd..07ce8e7 100644 --- a/logstash/handler_tcp.py +++ b/logstash/handler_tcp.py @@ -14,12 +14,12 @@ class TCPLogstashHandler(SocketHandler, object): :param tags: list of tags for a logger (default is None). """ - def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0): + def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0, **kwargs): super(TCPLogstashHandler, self).__init__(host, port) if version == 1: - self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn) + self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn, **kwargs) else: - self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn) + self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn, **kwargs) def makePickle(self, record): return self.formatter.format(record) + b'\n' From 48ea917ed33585fe1c1c2c814b3fd7958e36dc57 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:52:42 +0200 Subject: [PATCH 4/9] Add map for translating record attributes to output fields --- logstash/formatter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/logstash/formatter.py b/logstash/formatter.py index 9124750..fe5b188 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -10,6 +10,11 @@ import simplejson as json +# a mapping of attribute names on log record to output keys +# contains only those that need a mapping, fallback is record attribute name +field_map = { + 'name': 'logger', +} class LogstashFormatterBase(logging.Formatter): From c78f09edf174797c666a919dbe1778d482ef0784 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:54:23 +0200 Subject: [PATCH 5/9] Replace get_debug_fields with general get_fields method --- logstash/formatter.py | 58 ++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/logstash/formatter.py b/logstash/formatter.py index fe5b188..8bd8080 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -17,6 +17,20 @@ } +def format_field(record_key, value): + """ + Apply special formatting to certain record fields. + + :param record_key: record attribute name + :param value: attribute value to format + :return: the formatted value or original value + """ + if record_key == 'exc_info': + return ''.join(traceback.format_exception(*value)) if value else '' + + return value + + class LogstashFormatterBase(logging.Formatter): def __init__(self, message_type='Logstash', tags=None, fqdn=False, default_fields=None, exc_fields=None): @@ -70,23 +84,25 @@ def get_extra_fields(self, record): return fields - def get_debug_fields(self, record): - fields = { - 'exc_info': self.format_exception(record.exc_info), - 'lineno': record.lineno, - 'process': record.process, - 'threadName': record.threadName, - } - - # funcName was added in 2.5 - if not getattr(record, 'funcName', None): - fields['funcName'] = record.funcName - - # processName was added in 2.6 - if not getattr(record, 'processName', None): - fields['processName'] = record.processName - - return fields + @staticmethod + def get_fields(record, field_names): + """ + Get a dict with key/value pairs for all fields in `field_names` from + the `record`. Keys are translated according to the `field_map` and + special values formatted using `format_field()`. + + :param record: log record + :param field_names: list of record attribute names + :return: dict, ready for output + """ + return dict([ + ( + field_map.get(record_key, record_key), + format_field(record_key, getattr(record, record_key, None)) + ) + for record_key + in field_names + ]) @classmethod def format_source(cls, message_type, host, path): @@ -98,10 +114,6 @@ def format_timestamp(cls, time): return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % \ (tstamp.microsecond / 1000) + "Z" - @classmethod - def format_exception(cls, exc_info): - return ''.join(traceback.format_exception(*exc_info)) if exc_info else '' - @classmethod def serialize(cls, message): if sys.version_info < (3, 0): @@ -135,7 +147,7 @@ def format(self, record): # If exception, add debug info if record.exc_info: - message['@fields'].update(self.get_debug_fields(record)) + message['@fields'].update(self.get_fields(record, self.exc_fields)) return self.serialize(message) @@ -163,6 +175,6 @@ def format(self, record): # If exception, add debug info if record.exc_info: - message.update(self.get_debug_fields(record)) + message.update(self.get_fields(record, self.exc_fields)) return self.serialize(message) From 560633135c15ff3dbdf0b25816d6c678fe3358c4 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:54:51 +0200 Subject: [PATCH 6/9] Reuse get_fields for default fields --- logstash/formatter.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/logstash/formatter.py b/logstash/formatter.py index 8bd8080..6f20612 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -136,12 +136,11 @@ def format(self, record): '@source_path': record.pathname, '@tags': self.tags, '@type': self.message_type, - '@fields': { - 'levelname': record.levelname, - 'logger': record.name, - }, } + # Add default extra fields + message['@fields'].update(self.get_fields(record, self.default_fields)) + # Add extra fields message['@fields'].update(self.get_extra_fields(record)) @@ -164,12 +163,11 @@ def format(self, record): 'path': record.pathname, 'tags': self.tags, 'type': self.message_type, - - # Extra Fields - 'levelname': record.levelname, - 'logger': record.name, } + # Add default extra fields + message.update(self.get_fields(record, self.default_fields)) + # Add extra fields message.update(self.get_extra_fields(record)) From 55da2d239bc481d757d25751112a822f87c6cbab Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 18:55:04 +0200 Subject: [PATCH 7/9] Make method static --- logstash/formatter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logstash/formatter.py b/logstash/formatter.py index 6f20612..c53403c 100644 --- a/logstash/formatter.py +++ b/logstash/formatter.py @@ -59,7 +59,8 @@ def __init__(self, message_type='Logstash', tags=None, fqdn=False, else: self.host = socket.gethostname() - def get_extra_fields(self, record): + @staticmethod + def get_extra_fields(record): # The list contains all the attributes listed in # http://docs.python.org/library/logging.html#logrecord-attributes skip_list = ( @@ -152,7 +153,6 @@ def format(self, record): class LogstashFormatterVersion1(LogstashFormatterBase): - def format(self, record): # Create message dict message = { From b26e7e7defe060eafcbd5fca963139c4d7ce7c21 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 19:14:18 +0200 Subject: [PATCH 8/9] Pass through formatter kwargs with AMQP handler --- logstash/handler_amqp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logstash/handler_amqp.py b/logstash/handler_amqp.py index ad984db..7f9a6dd 100644 --- a/logstash/handler_amqp.py +++ b/logstash/handler_amqp.py @@ -40,8 +40,8 @@ class AMQPLogstashHandler(SocketHandler, object): def __init__(self, host='localhost', port=5672, username='guest', password='guest', exchange='logstash', exchange_type='fanout', virtual_host='/', message_type='logstash', tags=None, - durable=False, - version=0, extra_fields=True, fqdn=False, facility=None): + durable=False, version=0, extra_fields=True, fqdn=False, + facility=None, **kwargs): # AMQP parameters @@ -61,7 +61,7 @@ def __init__(self, host='localhost', port=5672, username='guest', self.tags = tags or [] fn = formatter.LogstashFormatterVersion1 if version == 1 \ else formatter.LogstashFormatterVersion0 - self.formatter = fn(message_type, tags, fqdn) + self.formatter = fn(message_type, tags, fqdn, **kwargs) # Standard logging parameters self.extra_fields = extra_fields From 9653c382ead70d1c4bef5a92f553576fdae317ac Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Wed, 1 Oct 2014 19:14:44 +0200 Subject: [PATCH 9/9] Update README --- README.rst | 69 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 91b60ad..42a9715 100644 --- a/README.rst +++ b/README.rst @@ -32,10 +32,13 @@ Using pip:: pip install python-logstash + Usage ===== -``LogstashHandler`` is a custom logging handler which sends Logstash messages using UDP. +``UDPLogstashHandler`` is a custom logging handler which sends JSON formatted +Logstash messages using UDP. ``TCPLogstashHandler`` provides the same +functionality via TCP. For example:: @@ -47,14 +50,24 @@ For example:: test_logger = logging.getLogger('python-logstash-logger') test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1)) + + test_logger.addHandler( + logstash.UDPLogstashHandler( + host, + 5959, + version=1, + default_fields=('levelname', 'name', 'lineno', 'funcName'), + ) + ) + + # alternatively use the TCP handler # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1)) test_logger.error('python-logstash: test logstash error message.') test_logger.info('python-logstash: test logstash info message.') test_logger.warning('python-logstash: test logstash warning message.') - # add extra field to logstash message + # add extra fields to logstash message extra = { 'test_string': 'python version: ' + repr(sys.version_info), 'test_boolean': True, @@ -65,12 +78,15 @@ For example:: } test_logger.info('python-logstash: test extra fields', extra=extra) -When using ``extra`` field make sure you don't use reserved names. From `Python documentation `_. +When using ``extra`` field make sure you don't use reserved names. +From `Python documentation `_. + | "The keys in the dictionary passed in extra should not clash with the keys used by the logging system. (See the `Formatter `_ documentation for more information on which keys are used by the logging system.)" -To use the AMQPLogstashHandler you will need to install pika first. - pip install pika +To use the ``AMQPLogstashHandler`` you will need to install pika first. + + pip install pika For example:: @@ -79,20 +95,26 @@ For example:: test_logger = logging.getLogger('python-logstash-logger') test_logger.setLevel(logging.INFO) - test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1)) + test_logger.addHandler( + logstash.AMQPLogstashHandler( + host='localhost', + version=1, + default_fields=('levelname', 'name', 'lineno', 'funcName') + ) + ) test_logger.info('python-logstash: test logstash info message.') try: 1/0 except: test_logger.exception('python-logstash-logger: Exception with stack trace!') - -Using with Django +Usage with Django ================= -Modify your ``settings.py`` to integrate ``python-logstash`` with Django's logging:: +Modify your ``settings.py`` to integrate ``python-logstash`` with Django's +logging:: LOGGING = { ... @@ -101,11 +123,23 @@ Modify your ``settings.py`` to integrate ``python-logstash`` with Django's loggi 'level': 'DEBUG', 'class': 'logstash.LogstashHandler', 'host': 'localhost', - 'port': 5959, # Default value: 5959 - 'version': 1, # Version of logstash event schema. Default value: 0 (for backward compatibility of the library) - 'message_type': 'logstash', # 'type' field in logstash message. Default value: 'logstash'. - 'fqdn': False, # Fully qualified domain name. Default value: false. - 'tags': ['tag1', 'tag2'], # list of tags. Default: None. + # default: 5959 + 'port': 5959, + # Version of logstash event schema, default: 0 (for backward compatibility of the library) + 'version': 1, + # 'type' field in logstash message, default: 'logstash' + 'message_type': 'logstash', + # Fully qualified domain name, default: False + 'fqdn': False, + # list of tags, default: None + 'tags': ['tag1', 'tag2'], + # log record attributes to include in the message, default: ('levelname', 'name') + 'default_fields': ( + 'levelname', + 'name', + 'lineno', + 'funcName', + ) }, }, 'loggers': { @@ -118,10 +152,13 @@ Modify your ``settings.py`` to integrate ``python-logstash`` with Django's loggi ... } + Contributors ------------ + - Volodymyr Klochan - Kiall Mac Innes - Alexander Boyd - joel-wright - - Dale O'Brien \ No newline at end of file + - Dale O'Brien + - Florian Demmer