Install prometheus from source code (Prometheus monitoring)

IP	Role	system	Specification
192.168.0.38	Prometheus server	CentOS 7	4c8g
192.168.0.25	node_exporter client	CentOS 7	4c8g

Prometheus download URL: Download | Prometheus

1. Download prometheus

[root@prometheus opt]#  wget https://github.com/prometheus/prometheus/releases/download/v2.47.2/prometheus-2.47.2.linux-amd64.tar.gz
[root@prometheus opt]# tar xvfz prometheus-2.47.2.linux-amd64.tar.gz -C /usr/local/
[root@prometheus opt]# cd /usr/local/
[root@prometheus local]# mv prometheus-2.47.2.linux-amd64 prometheus
[root@prometheus local]# cd prometheus
[root@prometheus prometheus]# ls
console_libraries  consoles  data  LICENSE  NOTICE  prometheus  prometheus.yml  promtool

2. Check version information

[root@prometheus prometheus]# ./prometheus --version
prometheus, version 2.47.2 (branch: HEAD, revision: 3f3172cde1ee37f1c7b3a5f3d9b031190509b3ad)
  build user:       root@79f2ad339b75
  build date:       20231012-16:07:10
  go version:       go1.21.3
  platform:         linux/amd64
  tags:             netgo,builtinassets,stringlabels

3. View help

[root@prometheus prometheus]# ./prometheus --help
prometheus, version 2.47.2 (branch: HEAD, revision: 3f3172cde1ee37f1c7b3a5f3d9b031190509b3ad)
  build user:       root@79f2ad339b75
  build date:       20231012-16:07:10
  go version:       go1.21.3
  platform:         linux/amd64
  tags:             netgo,builtinassets,stringlabels
  …………

4.prometheus.yml configuration file explanation

[root@prometheus prometheus]# cat prometheus.yml
# my global config
global:
  # 默认情况下，每15s拉取一次目标采样点数据。
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  # 每15秒评估一次规则。默认值为每1分钟。
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # job名称会增加到拉取到的所有采样点上，同时还有一个instance目标服务的host：port标签也会增加到采样点上
  - job_name: 'prometheus'

    # 覆盖global的采样点，拉取时间间隔5s
    scrape_interval: 5s
    static_configs:
    - targets: ['localhost:9090']

5. Start prometheus

[root@prometheus prometheus]# ./prometheus --config.file=prometheus.yml

6. Specifiable startup parameters

# 指定配置文件
--config.file="prometheus.yml"
# 默认指定监听地址端口，可修改端口
--web.listen-address="0.0.0.0:9090" 
# 最大连接数
--web.max-connections=512
# tsdb数据存储的目录，默认当前data/
--storage.tsdb.path="data/"
# premetheus 存储数据的时间，默认保存15天
--storage.tsdb.retention=15d 
# 通过命令热加载无需重启 curl -XPOST 192.168.2.45:9090/-/reload
--web.enable-lifecycle
# 可以启用 TLS 或 身份验证 的配置文件的路径
--web.config.file=""

启动选项了解：./prometheus --help

7.访问：http://localhost:9090

8. View exposure metricshttp://localhost:9090/metrics

9. Configure Prometheus as systemd management

# 配置Prometheus启动文件
[root@prometheus opt]# vim /usr/lib/systemd/system/prometheus.service
[Unit]
Description=https://prometheus.io

[Service]
Restart=on-failure
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml --web.listen-address=:9090

[Install]                      
WantedBy=multi-user.target

# 重新加载system配置
[root@prometheus opt]# systemctl daemon-reload
[root@prometheus opt]# systemctl start prometheus
[root@prometheus opt]# ss -tlanp |grep 9090
LISTEN     0      1024      [::]:9090                  [::]:*                   users:(("prometheus",pid=9318,fd=7))

10. Client, configure service discovery to monitor Linux hosts and related services

# 安装node_exporter
[root@node ~]# wget https://github.com/prometheus/node_exporter/releases/download/v1.7.0/node_exporter-1.7.0.linux-amd64.tar.gz
[root@node ~]# tar -xr node_exporter-1.7.0.linux-amd64.tar.gz -C /usr/local/
[root@node ~]# cd /usr/local/
[root@node local]# mv node_exporter-1.7.0.linux-amd64/ node_exporter
[root@node local]# cd node_exporter/
[root@node node_exporter]# ls
LICENSE  node_exporter  NOTICE
[root@node node_exporter]# ./node_exporter &
    …………
ts=2023-11-16T09:52:04.979Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100
ts=2023-11-16T09:52:04.979Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100

[root@node node_exporter]# ss -tlnp  | grep 9100
LISTEN     0      1024      [::]:9100                  [::]:*                   users:(("node_exporter",pid=8414,fd=3))

11. Configure node_exporter to be managed by systemd

[root@node node_exporter]# vim /usr/lib/systemd/system/node_exporter.service
[Unit]
Description=node_exporter
After=network.target 

[Service]
ExecStart=/usr/local/node_exporter/node_exporter
Restart=on-failure

[Install]
WantedBy=multi-user.target

[root@node node_exporter]# systemctl daemon-reload
[root@node node_exporter]# systemctl start node_exporter
[root@node node_exporter]# ss -tlnp  | grep 9100
LISTEN     0      1024      [::]:9100                  [::]:*                   users:(("node_exporter",pid=8675,fd=3))

12. Prometheus`服务端`Add monitoring items to configuration file

[root@prometheus prometheus]# vim prometheus.yml 
……
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]
  # 添加以下内容
  - job_name: "node_exporter"
    static_configs:
    # 如果有多个机器，用','分开
    - targets: ['192.168.0.25:9100']

# 重启prometheus服务
[root@prometheus prometheus]# systemctl restart prometheus

13. Monitor mysql (mysqld-exporter)

# 安装mysqld-exporter
[root@node ~]# wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.15.0/mysqld_exporter-0.15.0.linux-amd64.tar.gz
[root@node ~]# tar xf mysqld_exporter-0.15.0.linux-amd64.tar.gz -C /usr/local/
[root@node /usr/local]# mv mysqld_exporter-0.15.0.linux-amd64 mysqld_exporter
[root@node /usr/local/mysqld_exporter]# vim .my.cnf
[client]
user=root
password=123456

# 启动mysqld-exporter
[root@node /usr/local/mysqld_exporter]# ./mysqld_exporter --config.my-cnf="/usr/local/mysqld_exporter/.my.cnf" &
[root@node /usr/local/mysqld_exporter]# ps -ef |grep mysqld_exporter
root       3447   3398  0 01:31 pts/1    00:00:02 ./node_exporter
root       4647   3398  0 02:13 pts/1    00:00:00 ./mysqld_exporter --config.my-cnf=/usr/local/mysqld_exporter/.my.cnf

[root@node /usr/local/mysqld_exporter]# ss -lntp |grep 4647
LISTEN     0      128         :::9104                    :::*                   users:(("mysqld_exporter",pid=4647,fd=3))
[root@VM_2-44 /usr/local/mysqld_exporter]# 
# 启动后会监听9104端口

14. Prometheus`服务端`Add monitoring items to configuration file

[root@prometheus prometheus]# vim prometheus.yml 
  - job_name: 'mysql'
    static_configs:
    - targets: ['192.168.2.44:9104']
[root@prometheus prometheus]# systemctl restart prometheus


# 获取内存
node_memory_MemTotal_bytes{job="node_exporter", instance="192.168.0.25:9100"}

15. Use Grafana to display Prometheus data

[root@prometheus ~]# wget https://mirrors.tuna.tsinghua.edu.cn/grafana/yum/rpm/Packages/grafana-10.0.0-1.x86_64.rpm
[root@prometheus ~]# yum install initscripts fontconfig -y
[root@prometheus ~]# yum install -y grafana-10.0.0-1.x86_64.rpm
[root@prometheus ~]# systemctl status grafana-server.service 
[root@prometheus ~]# ss -tlanp |grep grafana |grep LISTEN
LISTEN     0      1024      [::]:3000                  [::]:*                   users:(("grafana",pid=10629,fd=11))

# 启动后访问地址：ip:3000
# 初始用户名和密码都是admin