linux: awk

Awk is a line processor: Compared with the advantages of screen processing, there is no memory overflow or slow processing when processing large files. It is usually used to format text information.
Awk processing process: process each line in turn, and then output
Awk command form:
awk [-F|-f|-v] 'BEGIN{} //{command1; command2} END{}' file
[-F|-f|-v] Large parameter, -F specifies the separator, -f calls the script, -v defines the variable var=value
' ' refers to the code block
BEGIN initializes the code block, before processing each line, initializes the code, mainly refers to global variables, sets the FS separator
// to match the code block, you can Is a string or regular expression
{} command code block, containing one or more commands
; multiple commands are separated by semicolons
END ending code block, the code block executed after each line is processed, mainly for final calculation Or output summary information at the end.

Special points:
$0 means the entire current line
$1 The first field of each line
NF field quantity variable
NR The record number of each line, multi-file records increment
FNR Similar to NR, but multi-file records are not incremented, each file Both start at 1
\t tab
\n newline
FS BEGIN defines the separator
RS The input record separator, the default is a newline (that is, the text is input line by line)
~ Match, compared with == is not an exact comparison
! ~ does not match, inexact comparison
== equals, must All equal, exact comparison
!= not equal, exact comparison
&& logical and
|| logical or
+ means 1 or more when matched
/[0-9][0-9]+/ two or more numbers
/ [0-9][0-9]*/ One or more numbers
FILENAME file name
OFS output field separator, the default is also a space, can be changed to the
ORS output record separator such as tab, the default is a newline, that is The processing result is also output to the screen line by line
-F'[:#/]' defines three separators

print & $0
print is the main command for awk to print the specified content
awk '{print}' /etc/passwd == awk '{print $0}' /etc/passwd 
awk '{print " "}' /etc/passwd //Do not output the content of passwd, but output the same number of blank lines, which further explains that awk processes text line by line
awk '{print "a"}' /etc /passwd //Output the same number of a lines, one line has only one a letter
awk -F":" '{print $1}' /etc/passwd
awk -F: '{print $1; print $2}' /etc/passwd //The first two fields of each line are output in separate lines to further understand the processing text line by line
awk -F: '{print $1,$3,$6}' OFS="\t" /etc/passwd //Output field 1, 3,6,

specify the script file
awk -f script.awk file
BEGIN{
FS=":"
} {print $1} with tabs as separators -f
//The effect is the same as awk -F":" '{print $1} ' same, except that the delimiter is specified in the code itself using FS

awk 'BEGIN{X=0} /^$/{ X+=1 } END{print "I find",X, "blank lines."}' test
I find 4 blank lines.
ls -l|awk 'BEGIN{sum=0} !/^d/{sum+=$5} END{print "total size is",sum}' //Calculate the file size
total size is 17487

-F specifies the separator
$1 refers to After specifying the delimiter, the first field, $3 the third field, \t is the tab character
One or more consecutive spaces or tabs are regarded as a delimiter, that is, multiple spaces are regarded as a space
awk - F":" '{print $1}' /etc/passwd
awk -F":" '{print $1 $3}' /etc/passwd //$1 and $3 are connected to output, do not separate
awk -F":" '{print $1,$3}' /etc/passwd //One more comma, $1 and $3 are separated by spaces
awk -F":" '{print $1 " " $3}' /etc/passwd //Manually added between $1 and $3 Space separated
awk -F":" '{print "Username:" $1 "\t\t Uid:" $3 }' /etc/passwd //custom output 
awk -F: '{print NF}' /etc/passwd // show how many fields per line
awk -F: '{print $NF}' /etc/passwd //Print the value of the NFth field of each line
awk -F: 'NF==4 {print }' /etc/passwd //Display only 4 awk -F: 'NF>2{print $0}' /etc/passwd //Display lines with more than 2 fields
per line
awk '{print NR,$0}' /etc/passwd //Output each line The line number of
awk -F: '{print NR,NF,$NF,"\t",$0}' /etc/passwd //Print the line number, the number of fields, the last field value, the tab character, the content of each line in turn
awk -F: 'NR==5{print}' /etc/passwd //Display line 5
awk -F: 'NR==5 || NR==6{print}' /etc/passwd //Display line 5 Lines 5 and 6
route -n|awk 'NR!=1{print}' //Do not display the first line

//Match code block
//Pure character match!//Pure character mismatch ~//Field value match !   ~//Field value does not match ~/a1|a2/Field value matches a1 or a2
awk '/mysql/' /etc/passwd
awk '/mysql/{print }' /etc/passwd
awk '/mysql/{print $0}' /etc/passwd //The results of the three commands are the same
awk '!/mysql/{print $0}' /etc/passwd // Output lines that do not match mysql
awk '/mysql|mail/{print}' /etc/passwd
awk '!/mysql|mail/{print}' /etc/passwd
awk -F: '/mail/,/mysql/{ print}' /etc/passwd //Interval matching
awk '/[2][7][7]*/{print $0}' /etc/passwd //Match lines that start with 27, such as 27, 277, 2777...
awk -F: '$1~/mail/{print $1}' /etc/passwd //$1 matches the specified content to display
awk -F: '{if($1~/mail/) print $1}' / etc/passwd // same as above
awk -F: '$1!~/mail/{print $1}' /etc/passwd // does not match
awk -F: '$1!~/mail|mysql/{print $1}'        /etc/passwd

IF statement
must be used in {}, and the comparison content is expanded with ()
awk -F: '{if($1~/mail/) print $1}' /etc/passwd //shorthand
awk -F: '{if($1~/mail/) {print $1}}' /etc/passwd / /Write all
awk -F: '{if($1~/mail/) {print $1} else {print $2}}' /etc/passwd //if...else...conditional


expression
== != > >= 
awk -F":" '$1=="mysql"{print $3}' /etc/passwd 
awk -F":" '{if($1=="mysql") print $3}' /etc/passwd / /same as above
awk -F":" '$1!="mysql"{print $3}' /etc/passwd //not equal to
awk -F":" '$3>1000{print $3}' /etc/passwd / /greater than
awk -F":" '$3>=100{print $3}' /etc/passwd //greater than or equal to
awk -F":" '$3<1{print $3}' /etc/passwd //less than
awk -F":" '$3<=1{print $3}' /etc/passwd //Less than or equal to

logical operator
&& ||
awk -F: '$1~/mail/ && $3>8 {print }' /etc /passwd //Logical AND, $1 matches mail, and $3>8
awk -F: '{if($1~/mail/ && $3>8) print }' /etc/passwd
awk -F: '$1~/mail/ || $3>1000 {print }' /etc/passwd //Logical or
awk -F: '{if($1~/mail/ || $3>1000) print }' /etc/passwd

Numerical operation
awk -F: ' $3 > 100' /etc/passwd   
awk -F: '$3 > 100 || $3 < 5' /etc/passwd 
awk -F: '$3+$4 > 200' /etc/passwd
awk -F: '/mysql|mail /{print $3+10}' /etc/passwd //Add 10 to the third field to print
awk -F: '/mysql/{print $3-$4}' /etc/passwd //Subtraction
awk -F: '/mysql/{print $3*$4}' /etc/passwd //Multiplication
awk '/MemFree/{print $2/1024}' /proc/meminfo //Division
awk '/MemFree/{print int ($2/1024)}' /proc/meminfo //Round

output separator OFS
awk '$6 ~ /FIN/ || NR==1 {print NR,$4,$5,$6}' OFS="\t" netstat .txt
awk '$6 ~ /WAIT/ || NR==1 {print NR,$4,$5,$6}' OFS="\t" netstat.txt       
//Output field 6 matches WAIT lines, where each line is output number, fields 4, 5, 6, and use tabs to separate fields

Output processing results to a file
① Direct output in the command code block route -n|awk 'NR!=1{print > "./fs"}'  
② Use redirection for output route -n|awk 'NR!=1{print}' > ./fs

formatted output
netstat -anp|awk '{printf "%-8s %-8s %-10s\n",$1, $2,$3}'
printf means format output
% formatted output delimiter
-8 length is 8 characters
s means string type
Print the first three fields of each line, specify that the first field outputs the string type (length is 8), the second field outputs the string type (length is 8), and
the third field outputs the string type (length is 10)
netstat -anp|awk '$6=="LISTEN" || NR==1 {printf "%-10s %-10s %-10s \n",$1,$2,$3}'
netstat -anp|awk '$6== "LISTEN" || NR==1 {printf "%-3s %-10s %-10s %-10s \n",NR,$1,$2,$3}'

IF statement
awk -F: '{if($3>100 ) print "large"; else print "small"}' /etc/passwd
small
small
small
large
small
small
awk -F: 'BEGIN{A=0;B=0} {if($3>100) {A++; print " large"} else {B++; print "small"}} END{print A,"\t",B}' /etc/passwd
                                                                                                                  //ID is greater than 100, add 1 to A, otherwise add 1 to B
awk -F: '{if($3<100) next; else print}' /etc/passwd                         //小于100跳过，否则显示
awk -F: 'BEGIN{i=1} {if(i<NF) print NR,NF,i++ }' /etc/passwd  
awk -F: 'BEGIN{i=1} {if(i<NF) {print NR,NF} i++ }' /etc/passwd
另一种形式
awk -F: '{print ($3>100 ? "yes":"no")}'  /etc/passwd
awk -F: '{print ($3>100 ? $3":\tyes":$3":\tno")}'  /etc/passwd

while语句
awk -F: 'BEGIN{i=1} {while(i<NF) print NF,$i,i++}' /etc/passwd
7 root 1
7 x 2
7 0 3
7 0 4
7 root 5
7 /root 6

数组
netstat -anp|awk 'NR!=1{a[$6]++} END{for (i in a) print i,"\t",a[i]}'
netstat -anp|awk 'NR!=1{a[$6]++} END{for (i in a) printf "%-20s %-10s %-5s \n", i,"\t",a[ i]}'
9523 1    
9929 1    
LISTEN 6    
7903 1    
3038/cupsd 1    
7913 1    
10837 1    
9833 1    

apply 1
awk -F: '{print NF}' helloworld.sh //how many fields per line of the output file
awk -F: '{print $1,$2,$3,$4,$5}' helloworld.sh //output the first 5 fields
awk -F: '{print $1,$2,$3,$4,$5}' OFS='\t' helloworld.sh //Print the first 5 fields and use tab-separated output
awk -F: '{print NR, $1,$2,$3,$4,$5}' OFS='\t' helloworld.sh //Tab-separated output first 5 fields, and print line number

Apply 2
awk -F'[:#]' '{print NF}' helloworld.sh //Specify multiple separators: #, output how many fields per line
awk -F'[:#]' '{print $1,$2,$3,$4,$5,$6,$7}' OFS= '\t' helloworld.sh //Tab-separated output multi-field

application 3
awk -F'[:#/]' '{print NF}' helloworld.sh //Specify three separators and output each line of fields Count
awk -F'[:#/]' '{print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12}' helloworld. sh //Tab-separated output for multiple fields

Application 4
calculates the size of ordinary files in the /home directory, using KB as the unit
ls -l|awk 'BEGIN{sum=0} !/^d/{sum+=$5} END {print "total size is:",sum/1024,"KB"}'
ls -l|awk 'BEGIN{sum=0} !/^d/{sum+=$5} END{print "total size is:",int(sum/1024),"KB"}' //int is rounded What is the meaning of

application 5
statistics netstat -anp What is the number of connections with LISTEN and CONNECT status respectively
netstat -anp|awk '$6~/LISTEN|CONNECTED/{sum[$6]++} END{for (i in sum) printf " %-10s %-6s %-3s \n", i," ",sum[i]}'

Application 6
counts the total number of common files of different users in the /home directory?
ls -l|awk 'NR!=1 && !/^d/{sum[$3]++} END{for (i in sum) printf "%-6s %-5s %-3s \n",i," ",sum[i]}'  
mysql 199
root 374
What is the total size of the ordinary files of different users in the /home directory?
ls -l|awk 'NR!=1 && !/^d/{sum[$3]+=$5} END{for (i in sum) printf "%-6s %-5s %-3s %-2s \n" ,i," ",sum[i]/1024/1024,"MB"}'



awk 'BEGIN{math=0;eng=0;com=0;printf "Lineno.   Name    No.    Math   English   Computer    Total\n";printf "------------------------------------------------------------\n"}{math+=$3; eng+=$4; com+=$5;printf "%-8s %-7s %-7s %-7s %-9s %-10s %-7s \n",NR,$1,$2,$3,$4,$5,$3+$4+$5} END{printf "------------------------------------------------------------\n";printf "%-24s %-7s %-9s %-20s \n","Total:",math,eng,com;printf "%-24s %-7s %-9s %-20s \n","Avg:",math/NR,eng/NR,com/NR}' test0

[root@localhost home]# cat test0
Marry   2143 78 84 77
Jack    2321 66 78 45
Tom     2122 48 77 71
Mike    2537 87 97 95
Bob     2415 40 57 62