Log aggregation - up1/course-springboot-2024 GitHub Wiki

Logging with Spring Boot

Logging with Logback + JSON
ELK Stack
- Elasticsearch
- Logstash + Beat
- Kibana

Working with Spring 3.4.4

File application.properties

logging.structured.format.console=logstash

Write log

private static final Logger LOGGER = LoggerFactory.getLogger(FirstFilter.class);


LOGGER.atInfo()
  .addKeyValue("URI", request.getRequestURI())
 .log("First filter with " +  request.getRequestURI());

Working with Logback library

Add dependencies :: Logback JSON encoder and appenders

<dependency>
	<groupId>net.logstash.logback</groupId>
	<artifactId>logstash-logback-encoder</artifactId>
	<version>8.0</version>
</dependency>

Basic with JSON logback-spring.xml

<configuration>

    <appender name="jsonEncoder" class="ch.qos.logback.core.ConsoleAppender">
        <encoder class="ch.qos.logback.classic.encoder.JsonEncoder"/>
    </appender>

    <root level="info">
        <appender-ref ref="jsonEncoder"/>
    </root>

</configuration>

Write log

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

private final Logger logger = LoggerFactory.getLogger(EmployeeController.class);

logger.info("Handling getById request");

Advance File logback-spring.xml

<?xml version="1.0" encoding="UTF-8"?>
<configuration>

    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{dd-MM-yyyy HH:mm:ss.SSS} %magenta([%thread]) %highlight(%-5level) %logger{36}.%M - %msg%n</pattern>
        </encoder>
    </appender>

    <appender name="SAVE-TO-FILE" class="ch.qos.logback.core.FileAppender">
        <file>logs/application.log</file>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <Pattern>%d{dd-MM-yyyy HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M - %msg%n</Pattern>
        </encoder>
    </appender>

    <appender name="OUTBOUND_LOGS" class="ch.qos.logback.core.FileAppender">
        <file>logs/application-outbound.log</file>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <Pattern>%d{dd-MM-yyyy HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M - %msg%n</Pattern>
        </encoder>
    </appender>

    <logger name="com.example.demolog" additivity="false" level="info">
        <appender-ref ref="SAVE-TO-FILE" />
        <appender-ref ref="STDOUT" />
    </logger>

    <logger name="outbound-logs" additivity="false" level="info">
        <appender-ref ref="OUTBOUND_LOGS" />
        <appender-ref ref="STDOUT" />
    </logger>

    <root level="INFO">
        <appender-ref ref="STDOUT" />
    </root>
</configuration>

Create logger in Java class

From class name

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

private static final Logger log = LoggerFactory.getLogger(MyClass.class);

From logger name

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

private static final Logger log = LoggerFactory.getLogger("outbound-logs");

Logback with JSON format

Logback JSON encoder and appenders

Add dependency in file pom.xml

<dependency>
	<groupId>net.logstash.logback</groupId>
	<artifactId>logstash-logback-encoder</artifactId>
	<version>7.2</version>
</dependency>

Edit file logback-spring.xml

<?xml version="1.0" encoding="UTF-8"?>
<configuration>

    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{dd-MM-yyyy HH:mm:ss.SSS} %magenta([%thread]) %highlight(%-5level) %logger{36}.%M - %msg%n</pattern>
        </encoder>
    </appender>

    <appender name="SAVE-TO-FILE" class="ch.qos.logback.core.FileAppender">
        <file>logs/application.log</file>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <Pattern>%d{dd-MM-yyyy HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M - %msg%n</Pattern>
        </encoder>
    </appender>

    <appender name="OUTBOUND_LOGS" class="ch.qos.logback.core.FileAppender">
        <file>logs/application-outbound.log</file>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <Pattern>%d{dd-MM-yyyy HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M - %msg%n</Pattern>
        </encoder>
    </appender>

    <appender name="logstash" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>logs/logstash.json</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${LOG_FILE}.json.%d{yyyy-MM-dd}.gz</fileNamePattern>
            <maxHistory>7</maxHistory>
        </rollingPolicy>
        <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>UTC</timeZone>
                </timestamp>
                <pattern>
                    <pattern>
                        {
                        "timestamp": "@timestamp",
                        "severity": "%level",
                        "service": "${springAppName:-}",
                        "trace": "%X{traceId:-}",
                        "span": "%X{spanId:-}",
                        "baggage": "%X{key:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger{40}",
                        "rest": "%message"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
    </appender>

    <logger name="com.example.demolog" additivity="false" level="info">
        <appender-ref ref="SAVE-TO-FILE" />
        <appender-ref ref="STDOUT" />
    </logger>

    <logger name="outbound-logs" additivity="false" level="info">
        <appender-ref ref="OUTBOUND_LOGS" />
        <appender-ref ref="STDOUT" />
    </logger>

    <logger name="logstash" additivity="false" level="info">
        <appender-ref ref="logstash" />
    </logger>

    <root level="INFO">
        <appender-ref ref="STDOUT" />
    </root>
</configuration>