Linux运维故障排查总结

1. 服务端口是否打开或被占用，服务进程是否存在

#找到端口冲突的应用程序
[root@zabbix_server ~]#nc -l 80
Ncat: bind to :::80: Address already in use. QUITTING.

[root@zabbix_server ~]#ss -ntlp
State      Recv-Q Send-Q              Local Address:Port                             Peer Address:Port
LISTEN     0      128                             *:10050                                       *:*users:(("zabbix_agentd",pid=1307,fd=4),("zabbix_agentd",pid=1306,fd=4),("zabbix_agentd",pid=1305,fd=4),("zabbix_agentd",pid=1304,fd=4),("zabbix_agentd",pid=1303,fd=4),("zabbix_agentd",pid=1299,fd=4))
LISTEN     0      128                             *:10051                                       *:*users:(("zabbix_server",pid=1806,fd=4),("zabbix_server",pid=1805,fd=4),("zabbix_server",pid=1803,fd=4),("zabbix_server",pid=1802,fd=4),("zabbix_server",pid=1801,fd=4),("zabbix_server",pid=1800,fd=4),("zabbix_server",pid=1797,fd=4),("zabbix_server",pid=1796,fd=4),("zabbix_server",pid=1795,fd=4),("zabbix_server",pid=1794,fd=4),("zabbix_server",pid=1793,fd=4),("zabbix_server",pid=1792,fd=4),("zabbix_server",pid=1791,fd=4),("zabbix_server",pid=1789,fd=4),("zabbix_server",pid=1787,fd=4),("zabbix_server",pid=1786,fd=4),("zabbix_server",pid=1785,fd=4),("zabbix_server",pid=1784,fd=4),("zabbix_server",pid=1783,fd=4),("zabbix_server",pid=1782,fd=4),("zabbix_server",pid=1779,fd=4),("zabbix_server",pid=1778,fd=4),("zabbix_server",pid=1777,fd=4),("zabbix_server",pid=1776,fd=4),("zabbix_server",pid=1775,fd=4),("zabbix_server",pid=1773,fd=4),("zabbix_server",pid=1772,fd=4),("zabbix_server",pid=1771,fd=4),("zabbix_server",pid=1770,fd=4),("zabbix_server",pid=1769,fd=4),("zabbix_server",pid=1763,fd=4),("zabbix_server",pid=1762,fd=4),("zabbix_server",pid=1761,fd=4),("zabbix_server",pid=1759,fd=4),("zabbix_server",pid=1758,fd=4),("zabbix_server",pid=1757,fd=4),("zabbix_server",pid=1756,fd=4),("zabbix_server",pid=1755,fd=4),("zabbix_server",pid=1754,fd=4),("zabbix_server",pid=1753,fd=4),("zabbix_server",pid=1752,fd=4),("zabbix_server",pid=1751,fd=4),("zabbix_server",pid=1750,fd=4),("zabbix_server",pid=1749,fd=4),("zabbix_server",pid=1748,fd=4),("zabbix_server",pid=1747,fd=4),("zabbix_server",pid=1746,fd=4),("zabbix_server",pid=1744,fd=4),("zabbix_server",pid=1743,fd=4),("zabbix_server",pid=1742,fd=4),("zabbix_server",pid=1741,fd=4),("zabbix_server",pid=1740,fd=4),("zabbix_server",pid=1739,fd=4),("zabbix_server",pid=1735,fd=4))
LISTEN     0      128                     127.0.0.1:9000                                        *:*users:(("php-fpm",pid=1548,fd=9),("php-fpm",pid=1547,fd=9),("php-fpm",pid=1546,fd=9),("php-fpm",pid=1545,fd=9),("php-fpm",pid=1544,fd=9),("php-fpm",pid=1289,fd=6))
LISTEN     0      128                             *:22                                          *:*users:(("sshd",pid=1284,fd=3))
LISTEN     0      100                     127.0.0.1:25                                          *:*users:(("master",pid=1507,fd=13))
LISTEN     0      128                          [::]:10050                                    [::]:*users:(("zabbix_agentd",pid=1307,fd=5),("zabbix_agentd",pid=1306,fd=5),("zabbix_agentd",pid=1305,fd=5),("zabbix_agentd",pid=1304,fd=5),("zabbix_agentd",pid=1303,fd=5),("zabbix_agentd",pid=1299,fd=5))
LISTEN     0      128                          [::]:10051                                    [::]:*users:(("zabbix_server",pid=1806,fd=5),("zabbix_server",pid=1805,fd=5),("zabbix_server",pid=1803,fd=5),("zabbix_server",pid=1802,fd=5),("zabbix_server",pid=1801,fd=5),("zabbix_server",pid=1800,fd=5),("zabbix_server",pid=1797,fd=5),("zabbix_server",pid=1796,fd=5),("zabbix_server",pid=1795,fd=5),("zabbix_server",pid=1794,fd=5),("zabbix_server",pid=1793,fd=5),("zabbix_server",pid=1792,fd=5),("zabbix_server",pid=1791,fd=5),("zabbix_server",pid=1789,fd=5),("zabbix_server",pid=1787,fd=5),("zabbix_server",pid=1786,fd=5),("zabbix_server",pid=1785,fd=5),("zabbix_server",pid=1784,fd=5),("zabbix_server",pid=1783,fd=5),("zabbix_server",pid=1782,fd=5),("zabbix_server",pid=1779,fd=5),("zabbix_server",pid=1778,fd=5),("zabbix_server",pid=1777,fd=5),("zabbix_server",pid=1776,fd=5),("zabbix_server",pid=1775,fd=5),("zabbix_server",pid=1773,fd=5),("zabbix_server",pid=1772,fd=5),("zabbix_server",pid=1771,fd=5),("zabbix_server",pid=1770,fd=5),("zabbix_server",pid=1769,fd=5),("zabbix_server",pid=1763,fd=5),("zabbix_server",pid=1762,fd=5),("zabbix_server",pid=1761,fd=5),("zabbix_server",pid=1759,fd=5),("zabbix_server",pid=1758,fd=5),("zabbix_server",pid=1757,fd=5),("zabbix_server",pid=1756,fd=5),("zabbix_server",pid=1755,fd=5),("zabbix_server",pid=1754,fd=5),("zabbix_server",pid=1753,fd=5),("zabbix_server",pid=1752,fd=5),("zabbix_server",pid=1751,fd=5),("zabbix_server",pid=1750,fd=5),("zabbix_server",pid=1749,fd=5),("zabbix_server",pid=1748,fd=5),("zabbix_server",pid=1747,fd=5),("zabbix_server",pid=1746,fd=5),("zabbix_server",pid=1744,fd=5),("zabbix_server",pid=1743,fd=5),("zabbix_server",pid=1742,fd=5),("zabbix_server",pid=1741,fd=5),("zabbix_server",pid=1740,fd=5),("zabbix_server",pid=1739,fd=5),("zabbix_server",pid=1735,fd=5))
LISTEN     0      80                           [::]:3306                                     [::]:*users:(("mysqld",pid=1464,fd=21))
LISTEN     0      128                          [::]:80                                       [::]:*users:(("httpd",pid=1454,fd=4),("httpd",pid=1453,fd=4),("httpd",pid=1452,fd=4),("httpd",pid=1451,fd=4),("httpd",pid=1450,fd=4),("httpd",pid=1288,fd=4))
LISTEN     0      128                          [::]:22                                       [::]:*users:(("sshd",pid=1284,fd=4))
LISTEN     0      100                         [::1]:25                                       [::]:*users:(("master",pid=1507,fd=14))

[root@zabbix_server ~]#lsof -i:80
COMMAND  PID   USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
httpd   1288   root    4u  IPv6  23480      0t0  TCP *:http (LISTEN)
httpd   1450 apache    4u  IPv6  23480      0t0  TCP *:http (LISTEN)
httpd   1451 apache    4u  IPv6  23480      0t0  TCP *:http (LISTEN)
httpd   1452 apache    4u  IPv6  23480      0t0  TCP *:http (LISTEN)
httpd   1453 apache    4u  IPv6  23480      0t0  TCP *:http (LISTEN)
httpd   1454 apache    4u  IPv6  23480      0t0  TCP *:http (LISTEN)

#查看服务进程是否存在
[root@zabbix_server ~]#ps -ef | grep zabbix_server
root       1735      1  0 10:07 ?        00:00:00 /usr/sbin/zabbix_server -c /etc/zabbix/zabbix_server.conf
root       1739   1735  0 10:07 ?        00:00:02 /usr/sbin/zabbix_server: configuration syncer [synced configurationin 0.151211 sec, idle 60 sec]
root       1740   1735  0 10:07 ?        00:00:01 /usr/sbin/zabbix_server: alert manager #1 [sent 0, failed 0 alerts,idle 5.022430 sec during 5.022883 sec]
root       1741   1735  0 10:07 ?        00:00:00 /usr/sbin/zabbix_server: alerter #1 started
root       1742   1735  0 10:07 ?        00:00:00 /usr/sbin/zabbix_server: alerter #2 started
root       1743   1735  0 10:07 ?        00:00:00 /usr/sbin/zabbix_server: alerter #3 started
root       1744   1735  0 10:07 ?        00:00:05 /usr/sbin/zabbix_server: preprocessing manager #1 [queued 0, processed 12 values, idle 5.046546 sec during 5.048672 sec]
root       1746   1735  0 10:07 ?        00:00:01 /usr/sbin/zabbix_server: preprocessing worker #1 started
root       1747   1735  0 10:07 ?        00:00:00 /usr/sbin/zabbix_server: preprocessing worker #2 started
root       1748   1735  0 10:07 ?        00:00:00 /usr/sbin/zabbix_server: preprocessing worker #3 started

[root@zabbix_server ~]#pstree -p
           └─zabbix_server(1735)─┬─zabbix_server(1739)
                                 ├─zabbix_server(1740)
                                 ├─zabbix_server(1741)
                                 ├─zabbix_server(1742)
                                 ├─zabbix_server(1743)
                                 ├─zabbix_server(1744)
                                 ├─zabbix_server(1746)
                                 ├─zabbix_server(1747)
                                 ├─zabbix_server(1748)
                                 ├─zabbix_server(1749)
                                 ├─zabbix_server(1750)
                                 ├─zabbix_server(1751)
                                 ├─zabbix_server(1752)
                                 ├─zabbix_server(1753)
                                 ├─zabbix_server(1754)
                                 ├─zabbix_server(1755)
                                 ├─zabbix_server(1756)
                                 ├─zabbix_server(1757)
                                 ├─zabbix_server(1758)
                                 ├─zabbix_server(1759)
                                 ├─zabbix_server(1761)
                                 ├─zabbix_server(1762)
                                 ├─zabbix_server(1763)

2. 服务日志是否有报错error提示

[root@tomcat ~]#tail -f /usr/local/tomcat/logs/catalina.out
24-Jul-2022 10:09:41.214 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deploying web application directory [/usr/local/apache-tomcat-8.5.78/webapps/host-manager]
24-Jul-2022 10:09:41.767 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deployment of web application directory [/usr/local/apache-tomcat-8.5.78/webapps/host-manager] has finished in [554] ms
24-Jul-2022 10:09:41.769 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deploying web application directory [/usr/local/apache-tomcat-8.5.78/webapps/examples]
24-Jul-2022 10:09:46.694 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deployment of web application directory [/usr/local/apache-tomcat-8.5.78/webapps/examples] has finished in [4,925] ms
24-Jul-2022 10:09:46.696 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deploying web application directory [/usr/local/apache-tomcat-8.5.78/webapps/docs]
24-Jul-2022 10:09:46.986 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deployment of web application directory [/usr/local/apache-tomcat-8.5.78/webapps/docs] has finished in [289] ms
24-Jul-2022 10:09:46.988 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deploying web application directory [/usr/local/apache-tomcat-8.5.78/webapps/manager]
24-Jul-2022 10:09:47.283 INFO [localhost-startStop-1] org.apache.catalina.startup.HostConfig.deployDirectory Deployment of web application directory [/usr/local/apache-tomcat-8.5.78/webapps/manager] has finished in [294] ms
24-Jul-2022 10:09:47.312 INFO [main] org.apache.coyote.AbstractProtocol.start Starting ProtocolHandler ["http-nio-8080"]
24-Jul-2022 10:09:47.706 INFO [main] org.apache.catalina.startup.Catalina.start Server startup in 10847 ms

3. 服务运行状态问题

[root@tomcat ~]#systemctl status tomcat.service
● tomcat.service - Tomcat
   Loaded: loaded (/usr/lib/systemd/system/tomcat.service; enabled; vendor preset: disabled)
   Active: active (running) since Sun 2022-07-24 10:09:02 CST; 37min ago
  Process: 1292 ExecStart=/usr/local/tomcat/bin/startup.sh (code=exited, status=0/SUCCESS)
 Main PID: 1309 (java)
   CGroup: /system.slice/tomcat.service
           └─1309 /usr/local/jdk/bin/java -Djava.util.logging.config.file=/usr/local/tomcat/conf/logging.properties...

Jul 24 10:09:01 tomcat systemd[1]: Starting Tomcat...
Jul 24 10:09:02 tomcat startup.sh[1292]: Tomcat started.
Jul 24 10:09:02 tomcat systemd[1]: Started Tomcat.

4. 服务用户权限设置问题

[root@tomcat ~]#ll /usr/local/tomcat/
total 148
drwxr-x--- 2 tomcat tomcat  4096 Jun 28 10:11 bin
-rw-r----- 1 tomcat tomcat 19491 Apr  1 00:05 BUILDING.txt
drwx------ 3 tomcat tomcat  4096 Apr 24 23:50 conf
-rw-r----- 1 tomcat tomcat  6210 Apr  1 00:05 CONTRIBUTING.md
drwxr-x--- 2 tomcat tomcat  4096 Apr 24 23:42 lib
-rw-r----- 1 tomcat tomcat 57011 Apr  1 00:05 LICENSE
drwxr-x--- 2 tomcat tomcat  4096 Jul 24 10:09 logs
-rw-r----- 1 tomcat tomcat  1726 Apr  1 00:05 NOTICE
-rw-r----- 1 tomcat tomcat  3378 Apr  1 00:05 README.md
-rw-r----- 1 tomcat tomcat  7136 Apr  1 00:05 RELEASE-NOTES
-rw-r----- 1 tomcat tomcat 16497 Apr  1 00:05 RUNNING.txt
drwxr-x--- 2 tomcat tomcat  4096 Apr 24 23:42 temp
drwxr-x--- 7 tomcat tomcat  4096 Apr  1 00:05 webapps
drwxr-x--- 3 tomcat tomcat  4096 Apr 24 23:42 work

#常用命令
[root@tomcat ~]#chown -R tomcat.tomcat /usr/local/tomcat/*

[root@zabbix_server mysql]#ll /usr/local/mysql/
total 188
drwxr-xr-x  2 root root  4096 Oct 22  2020 bin
-rw-r--r--  1 root root 17987 Nov  6  2021 COPYING
-rw-r--r--  1 root root  2093 Nov  6  2021 CREDITS
drwxrwxr-x  2 root root  4096 Nov  6  2021 docs
drwxrwxr-x  3 root root  4096 Nov  6  2021 include
-rw-r--r--  1 root root  8779 Nov  6  2021 INSTALL-BINARY
drwxr-xr-x  5 root root  4096 Jul 29  2021 lib
drwxrwxr-x  4 root root  4096 Nov  6  2021 man
drwxrwxr-x 10 root root  4096 Feb 24 17:46 mysql-test
-rw-r--r--  1 root root  2874 Nov  6  2021 README.md
-rw-r--r--  1 root root 19520 Nov  6  2021 README-wsrep
drwxrwxr-x  2 root root  4096 Nov  6  2021 scripts
drwxrwxr-x 31 root root  4096 Nov  6  2021 share
drwxrwxr-x  4 root root  4096 Nov  6  2021 sql-bench
drwxrwxr-x  3 root root  4096 Nov  6  2021 support-files
-rw-r--r--  1 root root 86263 Nov  6  2021 THIRDPARTY

#常用命令
[root@zabbix_server mysql]#chmod guo+wr /var/lib/mysql/mysql.sock

5. 服务配置文件设置问题

#MySQL主从节点参数配置错误，会导致MySQL主从数据不一致
[root@master ~]#vim /etc/my.cnf
[mysqld]
server_id=7
binlog_format=row
log-bin=/data/logbin/mysql-bin

[root@slave ~]#vim /etc/my.cnf
[mysqld]
server_id=17
binlog_format=row
read_only=on
log-bin=/data/logbin/mysql-bin

6. 防火墙流入流出规则问题

[root@centos7 ~]# iptables -A INPUT -p tcp -m multiport --dport 21,22,23,80 -j ACCEPT
[root@centos7 ~]#iptables -A INPUT -j REJECT
[root@centos7 ~]#iptables -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
   12   872 ACCEPT     tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            multiport dports 21,22,23,80
    0     0 REJECT     all  --  *      *       0.0.0.0/0            0.0.0.0/0            reject-with icmp-port-unreach

7. 修改配置文件后，没有重新加载配置文件并重启服务

[root@centos7 local]#echo 'PATH=/usr/local/mysql/bin:$PATH' > /etc/profile.d/mysql.sh
[root@centos7 local]#. /etc/profile.d/mysql.sh

[root@ubuntu2004 ~]# . ~/.bashrc  #让.bashrc配置文件生效

#生成新的grub.cfg文件
[root@ubuntu2004 ~]#grub-mkconfig -o /boot/grub/grub.cfg
#重启生效
[root@ubuntu2004 ~]#reboot

[root@tomcat ~]#systemctl daemon-reload
[root@tomcat ~]#systemctl restart tomcat.service

8. SELINUX是否关闭

[root@ubuntu2004 ~]#getenforce  #获取selinux当前状态
Disabled

9 . 软件版本问题

#MySQL主从、redis主从、redis集群数据库版本要一致，zabbix proxy的版本必须和zabbix server版本一致，否则会导致出现zabbix proxy与zabbix server不兼容问题

10. 文件访问路径问题

Tomcat中默认网站根目录是$CATALINA_BASE/webapps/
在Tomcat中部署主站应用程序和其他应用程序，和之前WEB服务程序不同。

#假设有个网站：www.linux2022.com
#网页对应关系
/var/www/html/index.html  =>  http://www.linux2022.com/index.html
/var/www/html/test/index.html  =>  http://www.linux2022.com/test/index.html

/usr/local/tomcat/webapps/ROOT/index.jsp  =>  http://www.linux2022.com/index.jsp
/usr/local/tomcat/webapps/test/index.jsp  =>  http://www.linux2022.com/test/index.jsp

www.linux2022.com/test.html  =>  /usr/local/tomcat/webapps/ROOT/test.html
www.linux2022.com/blog/  =>  /usr/local/tomcat/webapps/blog/index.html

#目录对应关系：
www.linux2022.com/blog/   =>   /usr/local/tomcat/webapps/blog/  
www.linux2022.com/forum/   =>   /usr/local/tomcat/webapps/forum/ 

假设在nginx中部署2个网站应用eshop、forum，假设网站根目录是/data/nginx/html，那么部署可以是这样的。
eshop解压缩所有文件放到 /data/nginx/html/ 目录下，forum 的文件放在 /data/nginx/html/forum/ 下。
最终网站链接有以下对应关系

http://localhost/ 对应于eshop的应用，即 /data/nginx/html/
http://localhost/forum/ 对应于forum的应用，即/data/nginx/html/forum/

11. 网线是否插好，网络是否畅通

#如何判断网线连接状态
https://blog.csdn.net/weixin_51867896/article/details/122004900

#centos7 ping: www.baidu.com: Name or service not known的解决方法
https://blog.csdn.net/weixin_51867896/article/details/121692014

#判断对方主机是否在同一个网段
https://blog.csdn.net/weixin_51867896/article/details/122047558

12. 光盘镜像是否挂载成功

#自建yum仓库，分别为网络源和本地源
https://blog.csdn.net/weixin_51867896/article/details/121863542

Linux运维故障排查总结

Linux运维故障排查总结

1. 服务端口是否打开或被占用，服务进程是否存在

2. 服务日志是否有报错error提示

3. 服务运行状态问题

4. 服务用户权限设置问题

5. 服务配置文件设置问题

6. 防火墙流入流出规则问题

7. 修改配置文件后，没有重新加载配置文件并重启服务

8. SELINUX是否关闭

9 . 软件版本问题

10. 文件访问路径问题

11. 网线是否插好，网络是否畅通

12. 光盘镜像是否挂载成功

猜你喜欢