Make the fields included in messages configureable

README.rst

@@ -41,10 +41,13 @@ Using pip::
 
   pip install python-logstash
 
+
 Usage
 =====
 
-``LogstashHandler`` is a custom logging handler which sends Logstash messages using UDP.
+``UDPLogstashHandler`` is a custom logging handler which sends JSON formatted 
+Logstash messages using UDP. ``TCPLogstashHandler`` provides the same 
+functionality via TCP.
 
 For example::
 
@@ -56,14 +59,24 @@ For example::
 
   test_logger = logging.getLogger('python-logstash-logger')
   test_logger.setLevel(logging.INFO)
-  test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
+
+  test_logger.addHandler(
+      logstash.UDPLogstashHandler(
+          host, 
+          5959, 
+          version=1,
+          default_fields=('levelname', 'name', 'lineno', 'funcName'),
+      )
+  )
+
+  # alternatively use the TCP handler
   # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1))
 
   test_logger.error('python-logstash: test logstash error message.')
   test_logger.info('python-logstash: test logstash info message.')
   test_logger.warning('python-logstash: test logstash warning message.')
 
-  # add extra field to logstash message
+  # add extra fields to logstash message
   extra = {
       'test_string': 'python version: ' + repr(sys.version_info),
       'test_boolean': True,
@@ -74,12 +87,15 @@ For example::
   }
   test_logger.info('python-logstash: test extra fields', extra=extra)
 
-When using ``extra`` field make sure you don't use reserved names. From `Python documentation <https://docs.python.org/2/library/logging.html>`_.
+When using ``extra`` field make sure you don't use reserved names. 
+From `Python documentation <https://docs.python.org/2/library/logging.html>`_.
+
      | "The keys in the dictionary passed in extra should not clash with the keys used by the logging system. (See the `Formatter <https://docs.python.org/2/library/logging.html#logging.Formatter>`_ documentation for more information on which keys are used by the logging system.)"
 
-To use the AMQPLogstashHandler you will need to install pika first.
 
-   pip install pika
+To use the ``AMQPLogstashHandler`` you will need to install pika first.
+
+  pip install pika
 
 For example::
 
@@ -88,20 +104,26 @@ For example::
 
   test_logger = logging.getLogger('python-logstash-logger')
   test_logger.setLevel(logging.INFO)
-  test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1))
+  test_logger.addHandler(
+      logstash.AMQPLogstashHandler(
+          host='localhost',
+          version=1,
+          default_fields=('levelname', 'name', 'lineno', 'funcName')
+      )
+  )
 
   test_logger.info('python-logstash: test logstash info message.')
   try:
       1/0
   except:
       test_logger.exception('python-logstash-logger: Exception with stack trace!')
+
 
-
-
-Using with Django
+Usage with Django
 =================
 
-Modify your ``settings.py`` to integrate ``python-logstash`` with Django's logging::
+Modify your ``settings.py`` to integrate ``python-logstash`` with Django's 
+logging::
 
   LOGGING = {
     ...
@@ -110,11 +132,23 @@ Modify your ``settings.py`` to integrate ``python-logstash`` with Django's loggi
             'level': 'DEBUG',
             'class': 'logstash.LogstashHandler',
             'host': 'localhost',
-            'port': 5959, # Default value: 5959
-            'version': 1, # Version of logstash event schema. Default value: 0 (for backward compatibility of the library)
-            'message_type': 'logstash',  # 'type' field in logstash message. Default value: 'logstash'.
-            'fqdn': False, # Fully qualified domain name. Default value: false.
-            'tags': ['tag1', 'tag2'], # list of tags. Default: None.
+            # default: 5959
+            'port': 5959,
+            # Version of logstash event schema, default: 0 (for backward compatibility of the library)
+            'version': 1,
+            # 'type' field in logstash message, default: 'logstash'
+            'message_type': 'logstash',
+            # Fully qualified domain name, default: False
+            'fqdn': False,
+            # list of tags, default: None
+            'tags': ['tag1', 'tag2'],
+            # log record attributes to include in the message, default: ('levelname', 'name')
+            'default_fields': (
+              'levelname',
+              'name',
+              'lineno',
+              'funcName',
+            )
         },
     },
     'loggers': {

logstash/formatter.py

@@ -3,24 +3,52 @@
 import socket
 import sys
 from datetime import datetime
+
 try:
     import json
 except ImportError:
     import simplejson as json
 
 
-class LogstashFormatterBase(logging.Formatter):
+# a mapping of attribute names on log record to output keys
+# contains only those that need a mapping, fallback is record attribute name
+field_map = {
+    'name': 'logger',
+}
 
-    def __init__(self, message_type='Logstash', tags=None, fqdn=False):
+
+class LogstashFormatterBase(logging.Formatter):
+    def __init__(self, message_type='Logstash', tags=None, fqdn=False,
+            default_fields=None, exc_fields=None):
         self.message_type = message_type
         self.tags = tags if tags is not None else []
 
+        self.default_fields = default_fields \
+            if default_fields is not None \
+            else (
+                'levelname',
+                'name',
+            )
+        self.exc_fields = exc_fields \
+            if exc_fields is not None \
+            else (
+                'exc_info',
+                # funcName was added in 2.5
+                'funcName',
+                'lineno',
+                'process',
+                # processName was added in 2.6
+                'processName',
+                'threadName',
+            )
+
         if fqdn:
             self.host = socket.getfqdn()
         else:
             self.host = socket.gethostname()
 
-    def get_extra_fields(self, record):
+    @staticmethod
+    def get_extra_fields(record):
         # The list contains all the attributes listed in
         # http://docs.python.org/library/logging.html#logrecord-attributes
         skip_list = (
@@ -46,23 +74,38 @@ def get_extra_fields(self, record):
 
         return fields
 
-    def get_debug_fields(self, record):
-        fields = {
-            'stack_trace': self.format_exception(record.exc_info),
-            'lineno': record.lineno,
-            'process': record.process,
-            'thread_name': record.threadName,
-        }
-
-        # funcName was added in 2.5
-        if not getattr(record, 'funcName', None):
-            fields['funcName'] = record.funcName
-
-        # processName was added in 2.6
-        if not getattr(record, 'processName', None):
-            fields['processName'] = record.processName
-
-        return fields
+    @staticmethod
+    def get_fields(record, field_names):
+        """
+        Get a dict with key/value pairs for all fields in `field_names` from 
+        the `record`. Keys are translated according to the `field_map` and
+        special values formatted using `format_field()`.
+
+        :param record: log record
+        :param field_names: list of record attribute names
+        :return: dict, ready for output
+        """
+        return dict([
+            (
+                field_map.get(record_key, record_key),
+                self.format_field(record_key, getattr(record, record_key, None))
+            )
+            for record_key
+            in field_names
+        ])
+
+    def format_field(self, record_key, value):
+        """
+        Apply special formatting to certain record fields.
+
+        :param record_key: record attribute name
+        :param value: attribute value to format
+        :return: the formatted value or original value
+        """
+        if record_key == 'exc_info':
+            return self.format_exception(record.exc_info)
+
+        return value
 
     @classmethod
     def format_source(cls, message_type, host, path):
@@ -71,7 +114,8 @@ def format_source(cls, message_type, host, path):
     @classmethod
     def format_timestamp(cls, time):
         tstamp = datetime.utcfromtimestamp(time)
-        return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (tstamp.microsecond / 1000) + "Z"
+        return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % \
+            (tstamp.microsecond / 1000) + "Z"
 
     @classmethod
     def format_exception(cls, exc_info):
@@ -84,6 +128,7 @@ def serialize(cls, message):
         else:
             return bytes(json.dumps(message), 'utf-8')
 
+
 class LogstashFormatterVersion0(LogstashFormatterBase):
     version = 0
 
@@ -93,7 +138,7 @@ def format(self, record):
             '@timestamp': self.format_timestamp(record.created),
             '@message': record.getMessage(),
             '@source': self.format_source(self.message_type, self.host,
-                                          record.pathname),
+                record.pathname),
             '@source_host': self.host,
             '@source_path': record.pathname,
             '@tags': self.tags,
@@ -104,18 +149,20 @@ def format(self, record):
             },
         }
 
+        # Add default extra fields
+        message['@fields'].update(self.get_fields(record, self.default_fields))
+
         # Add extra fields
         message['@fields'].update(self.get_extra_fields(record))
 
         # If exception, add debug info
         if record.exc_info:
-            message['@fields'].update(self.get_debug_fields(record))
+            message['@fields'].update(self.get_fields(record, self.exc_fields))
 
         return self.serialize(message)
 
 
 class LogstashFormatterVersion1(LogstashFormatterBase):
-
     def format(self, record):
         # Create message dict
         message = {
@@ -132,11 +179,14 @@ def format(self, record):
             'logger_name': record.name,
         }
 
+        # Add default extra fields
+        message.update(self.get_fields(record, self.default_fields))
+
         # Add extra fields
         message.update(self.get_extra_fields(record))
 
         # If exception, add debug info
         if record.exc_info:
-            message.update(self.get_debug_fields(record))
+            message.update(self.get_fields(record, self.exc_fields))
 
         return self.serialize(message)

logstash/handler_amqp.py

@@ -47,7 +47,7 @@ def __init__(self, host='localhost', port=5672, username='guest',
                  password='guest', exchange='logstash', exchange_type='fanout',
                  virtual_host='/', message_type='logstash', tags=None,
                  durable=False, passive=False, version=0, extra_fields=True,
-                 fqdn=False, facility=None, exchange_routing_key=''):
+                 fqdn=False, facility=None, exchange_routing_key='', **kwargs):
 
 
         # AMQP parameters
@@ -68,7 +68,7 @@ def __init__(self, host='localhost', port=5672, username='guest',
         self.tags = tags or []
         fn = formatter.LogstashFormatterVersion1 if version == 1 \
             else formatter.LogstashFormatterVersion0
-        self.formatter = fn(message_type, tags, fqdn)
+        self.formatter = fn(message_type, tags, fqdn, **kwargs)
 
         # Standard logging parameters
         self.extra_fields = extra_fields

logstash/handler_tcp.py

@@ -14,12 +14,12 @@ class TCPLogstashHandler(SocketHandler, object):
     :param tags: list of tags for a logger (default is None).
     """
 
-    def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0):
+    def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0, **kwargs):
         super(TCPLogstashHandler, self).__init__(host, port)
         if version == 1:
-            self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn)
+            self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn, **kwargs)
         else:
-            self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn)
+            self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn, **kwargs)
 
     def makePickle(self, record):
         return self.formatter.format(record) + b'\n'