utils/third_party/nni_new/runtime/log.py

from datetime import datetime
from io import TextIOBase
import logging
from logging import FileHandler, Formatter, Handler, StreamHandler
from pathlib import Path
import sys
import time
from typing import Optional

import colorama

from .env_vars import dispatcher_env_vars, trial_env_vars


handlers = {}

log_format = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s'
time_format = '%Y-%m-%d %H:%M:%S'
formatter = Formatter(log_format, time_format)


def init_logger() -> None:
    """
    This function will (and should only) get invoked on the first time of importing nni (no matter which submodule).
    It will try to detect the running environment and setup logger accordingly.

    The detection should work in most cases but for `nnictl` and `nni.experiment`.
    They will be identified as "standalone" mode and must configure the logger by themselves.
    """
    colorama.init()

    if dispatcher_env_vars.SDK_PROCESS == 'dispatcher':
        _init_logger_dispatcher()
        return

    trial_platform = trial_env_vars.NNI_PLATFORM

    if trial_platform == 'unittest':
        return

    if trial_platform and not trial_env_vars.REUSE_MODE:
        _init_logger_trial()
        return

    _init_logger_standalone()

    logging.getLogger('filelock').setLevel(logging.WARNING)

_exp_log_initialized = False

def init_logger_experiment() -> None:
    """
    Initialize logger for `nni.experiment.Experiment`.

    This function will get invoked after `init_logger()`.
    """
    global _exp_log_initialized
    if not _exp_log_initialized:
        _exp_log_initialized = True
        colorful_formatter = Formatter(log_format, time_format)
        colorful_formatter.format = _colorful_format
        handlers['_default_'].setFormatter(colorful_formatter)

def start_experiment_log(experiment_id: str, log_directory: Path, debug: bool) -> None:
    log_path = _prepare_log_dir(log_directory) / 'dispatcher.log'
    log_level = logging.DEBUG if debug else logging.WARNING
    _register_handler(FileHandler(log_path), log_level, experiment_id)

def stop_experiment_log(experiment_id: str) -> None:
    if experiment_id in handlers:
        logging.getLogger().removeHandler(handlers.pop(experiment_id))


def _init_logger_dispatcher() -> None:
    log_level_map = {
        'fatal': logging.CRITICAL,
        'error': logging.ERROR,
        'warning': logging.WARNING,
        'info': logging.WARNING,
        'debug': logging.DEBUG,
        'trace': 0
    }

    log_path = _prepare_log_dir(dispatcher_env_vars.NNI_LOG_DIRECTORY) / 'dispatcher.log'
    log_level = log_level_map.get(dispatcher_env_vars.NNI_LOG_LEVEL, logging.WARNING)
    _register_handler(FileHandler(log_path), log_level)


def _init_logger_trial() -> None:
    log_path = _prepare_log_dir(trial_env_vars.NNI_OUTPUT_DIR) / 'trial.log'
    log_file = open(log_path, 'a')
    _register_handler(StreamHandler(log_file), logging.WARNING)

    if trial_env_vars.NNI_PLATFORM == 'local':
        sys.stdout = _LogFileWrapper(log_file)


def _init_logger_standalone() -> None:
    _register_handler(StreamHandler(sys.stdout), logging.WARNING)


def _prepare_log_dir(path: Optional[str]) -> Path:
    if path is None:
        return Path()
    ret = Path(path)
    ret.mkdir(parents=True, exist_ok=True)
    return ret

def _register_handler(handler: Handler, level: int, tag: str = '_default_') -> None:
    assert tag not in handlers
    handlers[tag] = handler
    handler.setFormatter(formatter)
    logger = logging.getLogger()
    logger.addHandler(handler)
    logger.setLevel(level)

def _colorful_format(record):
    time = formatter.formatTime(record, time_format)
    if not record.name.startswith('nni.'):
        return '[{}] ({}) {}'.format(time, record.name, record.msg % record.args)
    if record.levelno >= logging.ERROR:
        color = colorama.Fore.RED
    elif record.levelno >= logging.WARNING:
        color = colorama.Fore.YELLOW
    elif record.levelno >= logging.WARNING:
        color = colorama.Fore.GREEN
    else:
        color = colorama.Fore.BLUE
    msg = color + (record.msg % record.args) + colorama.Style.RESET_ALL
    if record.levelno < logging.WARNING:
        return '[{}] {}:{} {}'.format(time, record.threadName, record.name, msg)
    else:
        return '[{}] {}'.format(time, msg)

class _LogFileWrapper(TextIOBase):
    # wrap the logger file so that anything written to it will automatically get formatted

    def __init__(self, log_file: TextIOBase):
        self.file: TextIOBase = log_file
        self.line_buffer: Optional[str] = None
        self.line_start_time: Optional[datetime] = None

    def write(self, s: str) -> int:
        cur_time = datetime.now()
        if self.line_buffer and (cur_time - self.line_start_time).total_seconds() > 0.1:
            self.flush()

        if self.line_buffer:
            self.line_buffer += s
        else:
            self.line_buffer = s
            self.line_start_time = cur_time

        if '\n' not in s:
            return len(s)

        time_str = cur_time.strftime(time_format)
        lines = self.line_buffer.split('\n')
        for line in lines[:-1]:
            self.file.write(f'[{time_str}] PRINT {line}\n')
        self.file.flush()

        self.line_buffer = lines[-1]
        self.line_start_time = cur_time
        return len(s)

    def flush(self) -> None:
        if self.line_buffer:
            time_str = self.line_start_time.strftime(time_format)
            self.file.write(f'[{time_str}] PRINT {self.line_buffer}\n')
            self.file.flush()
            self.line_buffer = None