8 advanced Python logging features you shouldn't miss

Understand your program without sacrificing performance

Journaling is a very important part of software development. It helps developers better understand program execution and judge defects and unexpected failures. A log message can store information like the current status of a program or where it is running. If an error occurs, developers can quickly find the line of code that caused the problem and act accordingly.

Python logging . , .

logging

, , logging.

, , . logger = logging.getLogger(name). — name . name . , , .

, . , — (. .: formatter) (. .: handler).

. — LogRecord ( — ). , , , , , . :

 : :
# : WARNING:root: !

:

"%(asctime)s - [%(levelname)s] -  %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s"
# 2020-07-26 23:37:15,374 - [INFO] -  __main__ - (main.py).main(18) -  !

. . , logging . — FileHandler, , StreamHandler, , sys.stderr sys.stdout. . , sys.stderr. , .

, FileHandler WARNING (. .: ) StreamHandler INFO (. .: ). , sys.stdout, .

:



main.py, package1.py, app_logger.py. app_logger.py get_logger, . : StreamHandler INFO FileHandler WARNING. INFO DEBUG ( — WARNING), , WARNING, . main.py, package1.py, get_logger, .

Xiaoxu Gao

# app_logger.py
import logging

_log_format = f"%(asctime)s - [%(levelname)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s"

def get_file_handler():
    file_handler = logging.FileHandler("x.log")
    file_handler.setLevel(logging.WARNING)
    file_handler.setFormatter(logging.Formatter(_log_format))
    return file_handler

def get_stream_handler():
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.INFO)
    stream_handler.setFormatter(logging.Formatter(_log_format))
    return stream_handler

def get_logger(name):
    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)
    logger.addHandler(get_file_handler())
    logger.addHandler(get_stream_handler())
    return logger

# package1.py
import app_logger

logger = app_logger.get_logger(__name__)

def process(msg):
    logger.info(" ")
    print(msg)
    logger.info(" ")

# main.py
import package1
import app_logger

logger = app_logger.get_logger(__name__)

def main():
    logger.info(" ")
    package1.process(msg="")
    logger.warning("     ,     ")
    logger.info("  ")

if __name__ == "__main__":
    main()

# 2020-07-25 21:06:06,657 - [INFO] - __main__ - (main.py).main(8) -  
# 2020-07-25 21:06:06,658 - [INFO] - package1 - (package1.py).process(7) -  
# 
# 2020-07-25 21:06:06,658 - [INFO] - package1 - (package1.py).process(9) -  
# 2020-07-25 21:06:06,658 - [WARNING] - __main__ - (main.py).main(10) -      ,     
# 2020-07-25 21:06:06,658 - [INFO] - __main__ - (main.py).main(11) -   

# cat x.log
# 2020-07-25 21:06:06,658 - [WARNING] - __main__ - (main.py).main(10) -      ,     

basic-logging.py

INFO (sys.stdout), , WARNING . , , .

1. LogRecord, LoggerAdapter

, LogRecord . . , logging LogRecord .

— LoggerAdapter. , ( ). , Logger, logger.info.

- , , LoggerAdapter (. .: , -, ). . . app, , .

import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - [%(levelname)s] - %(app)s - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s",
)

logger = logging.getLogger(__name__)
logger = logging.LoggerAdapter(logger, {"app": " "})
logger.info(" ")
logger.info("  ")

# 2020-07-25 21:36:20,709 - [INFO] -   - __main__ - (main.py).main(8) -  
# 2020-07-25 21:36:20,709 - [INFO] -   - __main__ - (main.py).main(11) -   

logging_adapter_fixed_value.py

, , , - . LoggerAdapter . process() — , . id, . .

import logging

class CustomAdapter(logging.LoggerAdapter):
    def process(self, msg, kwargs):
        my_context = kwargs.pop('id', self.extra['id'])
        return '[%s] %s' % (my_context, msg), kwargs

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - [%(levelname)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s",
)

logger = logging.getLogger(__name__)
logger = CustomAdapter(logger, {"id": None})

logger.info('ID ', id='1234')
logger.info('ID ', id='5678')
logger.info('   ID')

# 2020-07-25 22:12:06,715 - [INFO] - __main__ - (main.py).<module>(38) - [1234] ID # 2020-07-25 22:12:06,715 - [INFO] - __main__ - (main.py).<module>(38) - [1234] ID 
# 2020-07-25 22:21:31,568 - [INFO] - __main__ - (main.py).<module>(39) - [5678] ID # 2020-07-25 22:21:31,568 - [INFO] - __main__ - (main.py).<module>(39) - [5678] ID 
# 2020-07-25 22:12:06,715 - [INFO] - __main__ - (main.py).<module>(39) - [None]    ID# 2020-07-25 22:12:06,715 - [INFO] - __main__ - (main.py).<module>(39) - [None]    ID

logging_adapter_dynamic_value.py

2. LogRecord, Filter

— Filter. , . , . , , filter().

Python

color (. .: ) filter(), . .

import logging

class CustomFilter(logging.Filter):

    COLOR = {
        "DEBUG": "GREEN",
        "INFO": "GREEN",
        "WARNING": "YELLOW",
        "ERROR": "RED",
        "CRITICAL": "RED",
    }

    def filter(self, record):
        record.color = CustomFilter.COLOR[record.levelname]
        return True

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s - [%(levelname)s] - [%(color)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s",
)

logger = logging.getLogger(__name__)
logger.addFilter(CustomFilter())

logger.debug("  ,  — ")
logger.info(" ,  — ")
logger.warning(" ,  — ")
logger.error("  ,  — ")
logger.critical("   ,  — ")

# 2020-07-25 22:45:17,178 - [DEBUG] - [GREEN] - __main__ - (main.py).<module>(52) -   ,  — 
# 2020-07-25 22:45:17,179 - [INFO] - [GREEN] - __main__ - (main.py).<module>(53) -  ,  — 
# 2020-07-25 22:45:17,179 - [WARNING] - [YELLOW] - __main__ - (main.py).<module>(54) -  ,  — 
# 2020-07-25 22:45:17,179 - [ERROR] - [RED] - __main__ - (main.py).<module>(55) -   ,  — 
# 2020-07-25 22:45:17,179 - [CRITICAL] - [RED] - __main__ - (main.py).<module>(56) -    ,  — 

logging_filter_dynamic_attributes.py

3. logging

logging , . , MainThread WorkThread . threadName .

import logging
import threading

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - [%(levelname)s] - [%(threadName)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s",
)
logger = logging.getLogger(__name__)

def worker():
    for i in range(5):
        logger.info(f"  {i}   ")

thread = threading.Thread(target=worker)
thread.start()

for i in range(5):
    logger.info(f"  {i}   ")

thread.join()

# 2020-07-26 15:33:21,078 - [INFO] - [Thread-1] - __main__ - (main.py).worker(62) - 0-     
# 2020-07-26 15:33:21,078 - [INFO] - [Thread-1] - __main__ - (main.py).worker(62) - 1-     
# 2020-07-26 15:33:21,078 - [INFO] - [Thread-1] - __main__ - (main.py).worker(62) - 2-     
# 2020-07-26 15:33:21,078 - [INFO] - [Thread-1] - __main__ - (main.py).worker(62) - 3-     
# 2020-07-26 15:33:21,078 - [INFO] - [Thread-1] - __main__ - (main.py).worker(62) - 4-     
# 2020-07-26 15:33:21,078 - [INFO] - [MainThread] - __main__ - (main.py).<module>(69) - 0-     
# 2020-07-26 15:33:21,078 - [INFO] - [MainThread] - __main__ - (main.py).<module>(69) - 1-     
# 2020-07-26 15:33:21,078 - [INFO] - [MainThread] - __main__ - (main.py).<module>(69) - 2-     
# 2020-07-26 15:33:21,078 - [INFO] - [MainThread] - __main__ - (main.py).<module>(69) - 3-     
# 2020-07-26 15:33:21,078 - [INFO] - [MainThread] - __main__ - (main.py).<module>(69) - 4-     

logging_multi_threading.py

logging threading.RLock() . RLock Lock:

1. Lock , . , RLock , .

   
   
   

2. Lock , RLock — , .

   
   
   

, Handler, handle(), . Handler.handle(). , . emit() - .

def handle(self, record):
    """
         .
       ,      .
        /
      -.  ,   ,     
    .
    """
    rv = self.filter(record)
    if rv:
        self.acquire()
        try:
            self.emit(record)
        finally:
            self.release()
    return rv

logging_handle.py

4. logging — QueueHandler

, logging , . , , . logging, .

QueueHandler + «-»

— QueueHandler. , multiprocessing.Queue . 2 «-», «-», .

, , , log_processor logger.log(record.levelno, record.msg) logger.info() logger.warning(). (. .: main) , log_processor . — , logging .

import os
from logging.handlers import QueueHandler

formatter = "%(asctime)s - [%(levelname)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s"
logging.basicConfig(level=logging.INFO)

def log_processor(queue):
    logger = logging.getLogger(__name__)
    file_handler = logging.FileHandler("a.log")
    file_handler.setFormatter(logging.Formatter(formatter))
    logger.addHandler(file_handler)

    while True:
        try:
            record = queue.get()
            if record is None:
                break
            logger.log(record.levelno, record.msg)
        except Exception as e:
            pass

def log_producer(queue):
    pid = os.getpid()
    logger = logging.getLogger(__name__)
    queue_handler = QueueHandler(queue)
    logger.addHandler(QueueHandler(queue))

    logger.info(f" INFO -  {pid}")
    logger.warning(f" WARNING -  {pid}")

def main():
    queue = multiprocessing.Queue(-1)
    listener = multiprocessing.Process(target=log_processor, args=(queue,))
    listener.start()
    workers = []
    for i in range(2):
        worker = multiprocessing.Process(target=log_producer, args=(queue,))
        workers.append(worker)
        worker.start()
    for w in workers:
        w.join()
    queue.put_nowait(None)
    listener.join()

if __name__ == '__main__':
    main()

# >> cat a.log 
# 2020-07-26 18:38:10,525 - [INFO] - __mp_main__ - (main.py).log_processer(118) -  INFO -  32242
# 2020-07-26 18:38:10,525 - [WARNING] - __mp_main__ - (main.py).log_processer(118) -  WARNING -  32242
# 2020-07-26 18:38:10,530 - [INFO] - __mp_main__ - (main.py).log_processer(118) -  INFO -  32243
# 2020-07-26 18:38:10,530 - [WARNING] - __mp_main__ - (main.py).log_processer(118) -  WARNING -  32243

logging_queue_handler.py

QueueHandler + QueueListener

logging.handlers QueueListener. . QueueListener , , listener. .

def log_producer(queue):
    pid = os.getpid()
    logger = logging.getLogger(__name__)
    logger.addHandler(QueueHandler(queue))

    logger.info(f" INFO -  {pid}")
    logger.warning(f" WARNING -  {pid}")

def main():
    queue = multiprocessing.Queue(-1)
    file_handler = logging.FileHandler("b.log")
    file_handler.setFormatter(logging.Formatter(formatter))
    queue_listener = QueueListener(queue, file_handler)
    queue_listener.start()

    workers = []
    for i in range(2):
        worker = multiprocessing.Process(target=log_producer, args=(queue,))
        workers.append(worker)
        worker.start()
    for w in workers:
        w.join()
    queue_listener.stop()

if __name__ == '__main__':
    main()

# >> cat b.log 
# 2020-07-26 20:15:58,656 - [INFO] - __mp_main__ - (main.py).log_producer(130) -  INFO -  34199
# 2020-07-26 20:15:58,656 - [WARNING] - __mp_main__ - (main.py).log_producer(131) -  WARNING -  34199
# 2020-07-26 20:15:58,662 - [INFO] - __mp_main__ - (main.py).log_producer(130) -  INFO -  34200
# 2020-07-26 20:15:58,662 - [WARNING] - __mp_main__ - (main.py).log_producer(131) -  WARNING -  34200

logging_queue_listener.py

SocketHandler

, , — SocketHandler , -, . .

, , : — , . .

5. - — NullHandler

, logging.

— NullHandler. NullHandler . , .

NullHandler.

class NullHandler(Handler):
    """
        .   ,  
      "  XXX    ". 
       ,       .  
        ,   
      ;   ,       
    NullHandler          
    .
    """

    def handle(self, record):
        """."""

    def emit(self, record):
        """."""

    def createLock(self):
        self.lock = None

logging_nullhandler.py

?

logging, Vinay Sajip:

, logging, , / , .

— , .

init.py, NullHandler. . pip install, .

# package/
# │
# ├── __init__.py
# └── module1.py

# __init__.py
import logging
logging.getLogger(__name__).addHandler(logging.NullHandler())

# module1.py
logger = logging.getLogger(__name__)
logger.info("   ")

logging_nullhandler_example.py

, .

#   
logging.getLogger("package").addHandler(logging.StreamHandler())

NullHandler, , logging.getLogger("package").propagate = False. propagate False, .

6. — RotatingFileHandler, TimedRotatingFileHandler

RotatingFileHandler , . : maxBytes backupCount. maxBytes , . backupCount — . «» «.1», «.2» . - , .

. 6 .

import logging
from logging.handlers import RotatingFileHandler

def create_rotating_log(path):
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    handler = RotatingFileHandler(path, maxBytes=20, backupCount=5)
    logger.addHandler(handler)

    for i in range(100):
        logger.info("  -   %s" % i)

if __name__ == "__main__":
    log_file = "test.log"
    create_rotating_log(log_file)

logging_file_rotation.py

— TimeRotatingFileHandler, , . : , , , , (0=) ( ).

. .

import logging
import time
from logging.handlers import TimedRotatingFileHandler

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

handler = TimedRotatingFileHandler(
    "timed_test.log", when="s", interval=1, backupCount=5
)
logger.addHandler(handler)
for i in range(6):
    logger.info(i)
    time.sleep(1)

time_file_rotation.py

7.

logger.error() logger.exception() . ? ? , .

, emit(). , , , , . , handleError() stderr, . , Handler, handleError().

def emit(self, record):
    """
     .
      ,     .
             . 
       ,     
    traceback.print_exception     .   
      'encoding',     ,  
      .
    """
    try:
        msg = self.format(record)
        stream = self.stream
        # issue 35046: merged two stream.writes into one.
        stream.write(msg + self.terminator)
        self.flush()
    except RecursionError:  # See issue 36272
        raise
    except Exception:
        self.handleError(record)

logging_emit.py

. , .

import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - [%(levelname)s] -  %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s",
)

logger = logging.getLogger(__name__)

logger.info(" ")
logger.info("   %s   ", "msg", "other")
logger.info("  ")

# 2020-07-26 23:37:15,373 - [INFO] -  __main__ - (main.py).main(16) -  
# --- Logging error ---
# Traceback (most recent call last):
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 1197, in emit
#     msg = self.format(record)
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 1032, in format
#     return fmt.format(record)
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 756, in format
#     record.message = record.getMessage()
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 443, in getMessage
#     msg = msg % self.args
# TypeError: not all arguments converted during string formatting (. .:       )
# Call stack:
#   File "/Users/ie15qx/repo/compare_xml_files/source/main.py", line 22, in <module>
#     main()
#   File "/Users/ie15qx/repo/compare_xml_files/source/main.py", line 17, in main
#     logger.info("   %s   ", "msg", "other")
# Message: '   %s   '
# Arguments: ('msg', 'other')
# 2020-07-26 23:37:15,374 - [INFO] -  __main__ - (main.py).main(18) -   

logging_error_handle.py

, emit(), , . , id logger.info LoggerAdapter. .

import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - [%(levelname)s] - %(app)s - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s",
)

logger = logging.getLogger(__name__)
logger = logging.LoggerAdapter(logger, {"app": " "})
logger.info(" ", id="123")
logger.info("  ")

# source/main.py
# Traceback (most recent call last):
#   File "/Users/ie15qx/repo/compare_xml_files/source/main.py", line 10, in <module>
#     logger.info(" ", id="123")
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 1960, in info
#     self.log(INFO, msg, *args, **kwargs)
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 2001, in log
#     self.logger.log(level, msg, *args, **kwargs)
#   File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py", line 1637, in log
#     self._log(level, msg, args, **kwargs)
# TypeError: _log() got an unexpected keyword argument 'id' (. .: _log()     'id')

logging_exception_raise.py

8.

, , — . .

— , , , . , (. .: ) .

use dictConfig

— logging.config.dictConfig, . JSON- . , , - .

import logging
import logging.config

LOGGING_CONFIG = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
    },
    'handlers': {
        'default': {
            'level': 'INFO',
            'formatter': 'standard',
            'class': 'logging.StreamHandler',
            'stream': 'ext://sys.stdout',
        },
    },
    'loggers': {
        '': {  # root logger
            'handlers': ['default'],
            'level': 'DEBUG',
            'propagate': True,
        }
    },
}
logging.config.dictConfig(LOGGING_CONFIG)

if __name__ == "__main__":
    log = logging.getLogger(__name__)
    log.info("hello world")

# 2020-07-28 21:44:09,966 [INFO] __main__: hello world

logging_configure_json.py

fileConfig

, , — logging.config.fileConfig .ini.

# logging_config.ini
# [loggers]
# keys=root

# [handlers]
# keys=stream_handler

# [formatters]
# keys=formatter

# [logger_root]
# level=DEBUG
# handlers=stream_handler

# [handler_stream_handler]
# class=StreamHandler
# level=DEBUG
# formatter=formatter
# args=(sys.stderr,)

# [formatter_formatter]
# format=%(asctime)s %(name)-12s %(levelname)-8s %(message)s

import logging
from logging.config import fileConfig

fileConfig('logging_config.ini')
logger = logging.getLogger()
logger.debug('hello world')

logging_configure_file.py

. , . , c = logging.config.listen(PORT) c.start(), .

, , logging, . -, , , !