Shell common examples (continuously updated)

1. Determine whether a certain string exists in the file, and add it if it does not exist

#!/usr/bin/env bash


strs=( "username soft nproc 65535" "username hard nproc 65535" )
file="/etc/security/limits.conf"
# 当数组中有空格时，数组引用需要用双引号包住
for str in "${strs[@]}"
do
# 判断 所需的字符串 是否在文件中存在如果存在则不进行操作
  is_exist=$(cat "${file}"|grep "${str}")
  if [ "${is_exist}"x == "x" ]; then
   echo "Ready to write ${str} to ${file}"
   echo "${str}" >> ${
    
    file}
   if [ $? == 0 ]; then
     echo "Write successfully"
   else
     echo "Write failure!"
   fi
  else
   echo "The ${str} already exists in the  ${file}"
  fi
done

 

2. Determine whether these ports are occupied under a certain machine

#!/usr/bin/env bash

ports=( 8030 9020 9030 9010 9060 8040 9050 8060 8000 )
for num in ${
    
    ports[@]}
do
  netstat -tunlp |grep $num
done

 

t: Display the connection status of the TCP transmission protocol
u: Display the connection status of the UDP transmission protocol.
n: Display ip and port as the program address, not the domain name.
l: Display the Socket that is listening
p: Display the pid and program name

Find the (TCP, UDP transport protocol) program being listened at this time, and display the address, pid and program name (in the format of ip+port)

 

3. Determine whether a class exists under the lib directory (including jar)

#!/usr/bin/env bash


ls /${
    
    hive_home}/lib | while read one_line
do
 class_name=$(jar -vtf $one_line |grep CustomInDBAuthenticationProviderImpl)
 if [[  ${
    
    class_name}x != "x"  ]]; then
   echo "jar：$one_line  contains the ${class_name}"
 fi
done

v: detailed output, that is, the fully qualified name of the class
t: display the contents of the jar, such as which classes are included, which files, etc.
f: need to specify the file name of the jar package

 

4. When the successful execution of the previous command affects the execution of the subsequent command

Determine whether the previous command was successfully executed. If not, the following logic will definitely not be executed successfully.

。。。
exit_code=$?
if [ $exit_code -ne 0 ];then
  exit $exit_code
fi
if  [  "${20}"  ]; then
    echo "starting to load data to mysql using mysql load"
    bash  $F_HOME/bin/load_function.sh  "${20}"
fi

 

5. Get the value of the yaml file

#!/usr/bin/env bash

declare -A config_map

while read line
do
  #行为空时跳过这行
  if  [  ! "${line}"  ]; then
    continue
  fi
  conf_name=$(echo ${
    
    line} | cut -d : -f 1 | sed 's/ //g' )
  conf_value=$(echo ${
    
    line} | cut -d : -f 2 |  sed 's/ //g' )
  # 如果值为空则设置默认值：bigdata
  config_map[${
    
    conf_name}]=${
    
    conf_value:-bigdata}
  echo "get config_map [ ${conf_name} : ${config_map[${conf_name}]} ]"
done < bigdata_conf.yaml

bigdata_conf.yaml file

zookeeper:
  zookeeper1: node1
  zookeeper2: node2
  zookeeper3: node3

hdfs:
  namenode1: node1
  namenode2: node2
  journalnode1: node1
  journalnode2: node2
  journalnode3: node3
# gao
yarn:
  resourcemanager1: node1
  resourcemanager2: node2
  jobhistorynode: node3

output

。。。
get config_map [ zookeeper : bigdata ]
get config_map [ zookeeper1 : node1 ]
get config_map [ zookeeper2 : node2 ]
get config_map [ zookeeper3 : node3 ]
。。。

 

6. shell retry script

#!/bin/sh
#记录重试次数
count=0     
# 重试标识，flag=0 表示任务正常，flag=1 表示需要进行重试
flag=0      
while [ 0 -eq 0 ]
do
    echo ".................. job begin  ..................."
    # ...... 添加要执行的内容，flag 的值在这个逻辑中更改为1，或者不变......
    
    sleep 10
    port_cnt=`netstat -anp | grep 9083 | grep LISTEN | wc -l`
    thread_cnt=`ps -ef | grep -v "grep " | grep "org.apache.hadoop.hive.metastore.HiveMetaStore" -c`
    if [[ $port_cnt -lt 1 || $thread_cnt -lt 1 ]];then
        flag=1
    else
        flag=0
    fi
    
    # 检查和重试过程   
    if [ flag -eq 0 ]; then     #执行成功，不重试
        echo "--------------- job complete ---------------"
        break;
    else                        #执行失败，重试
        count=$[${
    
    count}+1]
        if [ ${
    
    count} -eq 3 ]; then     #指定重试次数，重试超过4次即失败
            echo 'timeout,exit.'
            break
        fi
        echo "...............retry in 1 seconds .........."
        sleep 1
    fi
done