031. Prometheus - kimdonggwan337/dongdong GitHub Wiki

# mkdir /opt/monitoring/prometheus    > 외부 프로그램을 설치 파일

# cd /opt/monitoring/prometheus
# wget https://github.com/prometheus/prometheus/releases/download/v3.1.0/prometheus-3.1.0.linux-amd64.tar.gz
# tar zxvf prometheus-3.1.0.linux-amd64.tar.gz
# cd prometheus-3.1.0.linux-amd64

## config 문법 오류 검사
# ./promtool check config ./prometheus.yml

Checking ./prometheus.yml
 SUCCESS: ./prometheus.yml is valid prometheus config file syntax

## Prometheus user 생성

# useradd --no-create-home --shell /bin/false prometheus 
# cat /etc/passwd | grep "prometheus"
prometheus:x:1001:1001::/home/prometheus:/bin/false

## 디렉토리 권한 설정

# cd /opt
# chown -R prometheus:prometheus monitoring 
# cd /var/lib/
# chown -R prometheus:prometheus prometheus
# cd /etc
# chown -R prometheus:prometheus prometheus
# cd /usr/bin
# chown -R prometheus:prometheus prometheus

## prometheus를 관리할 디렉토리 생성 및 명령어 복사
# mkdir /var/lib/prometheus           >  파일, 로그 데이터 그리고 임시파일 같은 가변 데이터 파일
# mkdir /etc/prometheus               > 시스템의 부팅, 셧다운 시에 필요한 파일들과 시스템의 전반에 걸친 설정 파일들 및 초기 스크립트 파일
# mkdir /usr/bin/prometheus            
# mkdir /opt/monitoring/prometheus/prometheus-3.1.0.linux-amd64/consoles 
# mkdir /opt/monitoring/prometheus/prometheus-3.1.0.linux-amd64/console_libraries 


# cp ./prometheus /usr/bin/prometheus
# cp ./promtool /usr/bin/prometheus
# cp ./prometheus.yml /etc/prometheus/
# cp -r ./consoles /etc/prometheus
# cp -r ./console_libraries /etc/prometheus

## prometheus.service 파일 등록
# cd /etc/systemd/system
# vi prometheus.service

[Unit]
Description=Monitoring system and time series database
After=network-online.target

[Service]
User=prometheus
Type=simple
ExecStart=/usr/bin/prometheus \
  --config.file=/etc/prometheus/prometheus.yml \
  --storage.tsdb.path=/var/lib/prometheus \
  --web.console.templates=/etc/prometheus/consoles \
  --web.console.libraries=/etc/prometheus/console_libraries

[Install]
WantedBy=multi-user.target


# systemctl daemon-reload
# systemctl start prometheus

# ss -lntu
tcp                 LISTEN               0                    2048                                           *:9090                                           

 prometheus.service - Prometheus Server
   Loaded: loaded (/etc/systemd/system/prometheus.service; disabled; vendor preset: disabled)
   Active: active (running) since Wed 2025-02-12 07:42:12 UTC; 4min 28s ago
 Main PID: 18690 (prometheus)
    Tasks: 7 (limit: 22938)
   Memory: 28.0M
   CGroup: /system.slice/prometheus.service
           └─18690 /usr/bin/prometheus/prometheus --config.file /etc/prometheus/prometheus.yml --storage.tsdb.path /var/lib/prometheus --web.console.templates=/etc/prometheus/consoles --web.console.li>

      
# mkdir /opt/monitoring/node_exporter
# cd /opt/monitoring/node_exporter
# wget https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz
# tar xvfz node_exporter-1.2.2.linux-amd64.tar.gz

# cp ./node_exporter /usr/local/bin/
# useradd --no-create-home --shell /bin/false node_exporter

# vi /etc/systemd/system/node_exporter.service

[Unit]
Description=Node Exporter
After=network.target

[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
Environment="NODE_EXPORTER_ARGS=--web.listen-address=\":9100\" "
ExecStart=/usr/local/bin/node_exporter $NODE_EXPORTER_ARGS

[Install]
WantedBy=multi-user.target

# systemctl daemon-reload
# systemctl start node_exporter

### [Prometheus.yml]
 
# vi /etc/prometheus/prometheus.yml
------------------------------- 생략 -------------------------------

scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["20.41.83.18:9090"]


  - job_name: 'Suse_node-exporter1'
    scrape_interval: 5s

    static_configs:
       - targets: ["52.231.10.251:9100"]

  - job_name: 'Redhat_node-exporter2'
    scrape_interval: 5s

    static_configs:
       - targets: ["4.218.17.32:9100"]

---------------------------------------------------------------------

# mkdir /opt/monitoring/process_exporter/
# wget https://github.com/ncabatoff/process-exporter/releases/download/v0.8.4/process-exporter-0.8.4.linux-amd64.tar.gz
# tar xvfz process-exporter-0.8.4.linux-amd64.tar.gz
# cp  ./process-exporter /usr/local/bin/

# [process exporter config.yml]

process_names:
  - name: "{{.Matches}}"
    cmdline:
    - 'nginx'  # Nginx 프로세스 추적
  - name: "{{.Matches}}"
    cmdline:
    - 'sshd'   # SSH 프로세스 추적


# vi /etc/systemd/system/process-exporter.service

[Unit]
Description=Process Exporter
After=network.target

[Service]
User=process-exporter
ExecStart=/usr/local/bin/process-exporter -config.path /etc/process_exporter/config.yml
Restart=always

[Install]
WantedBy=multi-user.target

# useradd --no-create-home --shell /bin/false process-exporter
# chown -R process-exporter:process-exporter /etc/process-exporter
# chmod 640 /etc/process-exporter/config.yml

# systemctl daemon-reload
# systemctl start process-exporter

### [prometheus.yml]


  - job_name: 'Suse_process-exporter1'
    scrape_interval: 5s

    static_configs:
       - targets: ["52.231.10.251:9256"]


  - job_name: 'Redhat_process-exporter2'
    scrape_interval: 5s

    static_configs:
       - targets: ["4.218.17.32:9256"]

# firewall-cmd --permanent --add-port=9100/tcp
# firewall-cmd --reload

import time
import requests
import os

# Slack Webhook URL (Slack에서 발급받은 걸 넣어주세요)
SLACK_WEBHOOK_URL = "https://hooks.slack.com/services/XXXX/YYYY/ZZZZ"

# Prometheus 로그 파일 경로
PROMETHEUS_LOG = "/var/log/prometheus/prometheus.log"

def send_slack_alert(message: str):
    """Slack으로 알림 전송"""
    payload = {
        "text": f":rotating_light: Prometheus Error Detected!\n```\n{message}\n```"
    }
    try:
        response = requests.post(SLACK_WEBHOOK_URL, json=payload)
        if response.status_code != 200:
            print(f"Slack 전송 실패: {response.text}")
    except Exception as e:
        print(f"Slack 요청 중 오류: {e}")

def monitor_log():
    """로그 파일 모니터링 (tail -f)"""
    # 파일이 열릴 때까지 대기
    while not os.path.exists(PROMETHEUS_LOG):
        print("로그 파일을 찾을 수 없음. 대기 중...")
        time.sleep(5)

    with open(PROMETHEUS_LOG, "r") as f:
        # 기존 로그는 무시하고, 새로운 로그만 감시
        f.seek(0, os.SEEK_END)

        while True:
            line = f.readline()
            if not line:
                time.sleep(1)
                continue

            # error 포함된 로그만 필터링
            if "error" in line.lower():
                print(f"[ERROR DETECTED] {line.strip()}")
                send_slack_alert(line.strip())

if __name__ == "__main__":
    monitor_log()