Article directory
- File operation
- I/O operations
- file copy
- http
- bufio package
- ioutil package
- Concurrency
- Coroutine in Go language - Goroutine
- Concurrency Model of Go Language
- runtime package
- critical resource
- WaitGroup
- mutex
- read-write lock
- channel channel
- select statement
- CSP model of Go language
- reflection
File operation
First of all, the file class is in the os package, which encapsulates the underlying file descriptor and related information, and also encapsulates the implementation of Read and Write
FileInfo interface
The methods related to File information are defined in the FileInfo interface.
type FileInfo interface {
Name() string // base name of the file 文件名.扩展名 1.txt
Size() int64 // 文件大小，字节数 12540
Mode() FileMode // 文件权限 -rw-rw-rw-
ModTime() time.Time // 修改时间 2018-04-13 16:30:53 +0800 CST
IsDir() bool // 是否文件夹
Sys() interface{
} // 基础数据源接口(can return nil)
}
Sample code:
package main
import (
"fmt"
"os"
)
func main() {
fileInfo, err := os.Stat("main/1.md")
if err != nil {
fmt.Println("err :", err)
return
}
fmt.Printf("%T\n", fileInfo)
//文件名
fmt.Println(fileInfo.Name())
//文件大小
fmt.Println(fileInfo.Size())
//是否是目录
fmt.Println(fileInfo.IsDir()) //IsDirectory
//修改时间
fmt.Println(fileInfo.ModTime())
//权限
fmt.Println(fileInfo.Mode()) //-rw-r--r--
}
Output result:
*os.fileStat
1.md
47
false
2023-06-10 21:30:41.7576415 +0800 CST
-rw-rw-rw-
permissions
As for the operation permission perm, unless you need to specify it when creating a file, you can set it to 0 when you don’t need to create a new file. Although the go language sets many constants for perm permissions, it is customary to use numbers directly, such as 0666 (the specific meaning is consistent with that of the Unix system).
Access control:
There are two ways to express file permissions under linux, namely "symbolic representation" and "octal representation".
（1）符号表示方式:
- --- --- ---
type owner group others
文件的权限是这样子分配的 读 写 可执行 分别对应的是 r w x 如果没有那一个权限，用 - 代替
(-文件 d目录 |连接符号)
例如：-rwxr-xr-x
（2）八进制表示方式：
r ——> 004
w ——> 002
x ——> 001
- ——> 000
0755
0777（owner，group，others都是可读可写可执行）
0555
0444
0666
open mode
File open mode:
const (
O_RDONLY int = syscall.O_RDONLY // 只读模式打开文件
O_WRONLY int = syscall.O_WRONLY // 只写模式打开文件
O_RDWR int = syscall.O_RDWR // 读写模式打开文件
O_APPEND int = syscall.O_APPEND // 写操作时将数据附加到文件尾部
O_CREATE int = syscall.O_CREAT // 如果不存在将创建一个新文件
O_EXCL int = syscall.O_EXCL // 和O_CREATE配合使用，文件必须不存在
O_SYNC int = syscall.O_SYNC // 打开文件用于同步I/O
O_TRUNC int = syscall.O_TRUNC // 如果可能，打开时清空文件
)
File operation
type File
//File代表一个打开的文件对象。
func Create(name string) (file *File, err error)
//Create采用模式0666（任何人都可读写，不可执行）创建一个名为name的文件，如果文件已存在会截断它（为空文件）。如果成功，返回的文件对象可用于I/O；对应的文件描述符具有O_RDWR模式。如果出错，错误底层类型是*PathError。
func Open(name string) (file *File, err error)
//Open打开一个文件用于读取。如果操作成功，返回的文件对象的方法可用于读取数据；对应的文件描述符具有O_RDONLY模式。如果出错，错误底层类型是*PathError。
func OpenFile(name string, flag int, perm FileMode) (file *File, err error)
//OpenFile是一个更一般性的文件打开函数，大多数调用者都应用Open或Create代替本函数。它会使用指定的选项（如O_RDONLY等）、指定的模式（如0666等）打开指定名称的文件。如果操作成功，返回的文件对象可用于I/O。如果出错，错误底层类型是*PathError。
func NewFile(fd uintptr, name string) *File
//NewFile使用给出的Unix文件描述符和名称创建一个文件。
func Pipe() (r *File, w *File, err error)
//Pipe返回一对关联的文件对象。从r的读取将返回写入w的数据。本函数会返回两个文件对象和可能的错误。
func (f *File) Name() string
//Name方法返回（提供给Open/Create等方法的）文件名称。
func (f *File) Stat() (fi FileInfo, err error)
//Stat返回描述文件f的FileInfo类型值。如果出错，错误底层类型是*PathError。
func (f *File) Fd() uintptr
//Fd返回与文件f对应的整数类型的Unix文件描述符。
func (f *File) Chdir() error
//Chdir将当前工作目录修改为f，f必须是一个目录。如果出错，错误底层类型是*PathError。
func (f *File) Chmod(mode FileMode) error
//Chmod修改文件的模式。如果出错，错误底层类型是*PathError。
func (f *File) Chown(uid, gid int) error
//Chown修改文件的用户ID和组ID。如果出错，错误底层类型是*PathError。
func (f *File) Close() error
//Close关闭文件f，使文件不能用于读写。它返回可能出现的错误。
Sample code:
package main
import (
"fmt"
"os"
"path/filepath"
)
func main() {
//1.路径
fileName1 := "C:\\GolandProjects\\GoProject1\\main\\1.md"
fileName2 := "main/1.md"
//判断是否是绝对路径
fmt.Println(filepath.IsAbs(fileName1)) //true
fmt.Println(filepath.IsAbs(fileName2)) //false
//转化为绝对路径
//fmt.Println(filepath.Abs(fileName1))
//fmt.Println(filepath.Abs(fileName2)) // C:\GolandProjects\GoProject1\main\1.md <nil>
//1.获取目录
//fmt.Println("获取父目录：", filepath.Join(fileName1, ".."))
//fmt.Println("获取父目录：", filepath.Dir(fileName1))
//fmt.Println("获取当前目录：", filepath.Join(fileName1, "."))
//2.创建目录
err := os.Mkdir("main/app", os.ModePerm) //权限0777
if err != nil {
fmt.Println("err:", err)
return
}
fmt.Println("文件夹创建成功。。")
//err := os.MkdirAll("main/a/b/c", os.ModePerm)
//if err != nil {
// fmt.Println("err:", err)
// return
//}
//fmt.Println("多层文件夹创建成功")
//3.创建文件:Create采用模式0666（任何人都可读写，不可执行）创建一个名为name的文件，如果文件已存在会截断它（为空文件）
//file1, err := os.Create(fileName1)
//if err != nil {
// fmt.Println("err：", err)
// return
//}
//fmt.Println(file1)
//file2, err := os.Create(fileName2) //创建相对路径的文件，是以当前工程为参照的
//if err != nil {
// fmt.Println("err :", err)
// return
//}
//fmt.Println(file2)
//4.打开文件：
//file3, err := os.Open(fileName1) //只读的
//if err != nil {
// fmt.Println("err:", err)
// return
//}
//fmt.Println(file3)
/*
第一个参数：文件名称
第二个参数：文件的打开方式
const (
// Exactly one of O_RDONLY, O_WRONLY, or O_RDWR must be specified.
O_RDONLY int = syscall.O_RDONLY // open the file read-only.
O_WRONLY int = syscall.O_WRONLY // open the file write-only.
O_RDWR int = syscall.O_RDWR // open the file read-write.
// The remaining values may be or'ed in to control behavior.
O_APPEND int = syscall.O_APPEND // append data to the file when writing.
O_CREATE int = syscall.O_CREAT // create a new file if none exists.
O_EXCL int = syscall.O_EXCL // used with O_CREATE, file must not exist.
O_SYNC int = syscall.O_SYNC // open for synchronous I/O.
O_TRUNC int = syscall.O_TRUNC // truncate regular writable file when opened.
)
第三个参数：文件的权限：文件不存在创建文件，需要指定权限
*/
//file4, err := os.OpenFile(fileName1, os.O_RDONLY|os.O_WRONLY, os.ModePerm)
//if err != nil {
// fmt.Println("err:", err)
// return
//}
//fmt.Println(file4)
//5关闭文件，
//err := file4.Close()
//if err != nil {
// return
//}
//6.删除文件或文件夹：
//删除文件(该方法也可以删除空目录)
//err := os.Remove("main/1.md")
//if err != nil {
// fmt.Println("err:", err)
// return
//}
//fmt.Println("删除文件成功。。")
//删除目录
//err := os.RemoveAll("main/a/b/c")
//if err != nil {
// fmt.Println("err:", err)
// return
//}
//fmt.Println("删除目录成功。。
file read
Introduction to file operation functions and methods:
func (f *File) Readdir(n int) (fi []FileInfo, err error)
//Readdir读取目录f的内容，返回一个有n个成员的[]FileInfo，这些FileInfo是被Lstat返回的，采用目录顺序。对本函数的下一次调用会返回上一次调用剩余未读取的内容的信息。如果n>0，Readdir函数会返回一个最多n个成员的切片。这时，如果Readdir返回一个空切片，它会返回一个非nil的错误说明原因。如果到达了目录f的结尾，返回值err会是io.EOF。如果n<=0，Readdir函数返回目录中剩余所有文件对象的FileInfo构成的切片。此时，如果Readdir调用成功（读取所有内容直到结尾），它会返回该切片和nil的错误值。如果在到达结尾前遇到错误，会返回之前成功读取的FileInfo构成的切片和该错误。
func (f *File) Readdirnames(n int) (names []string, err error)
//Readdir读取目录f的内容，返回一个有n个成员的[]string，切片成员为目录中文件对象的名字，采用目录顺序。对本函数的下一次调用会返回上一次调用剩余未读取的内容的信息。如果n>0，Readdir函数会返回一个最多n个成员的切片。这时，如果Readdir返回一个空切片，它会返回一个非nil的错误说明原因。如果到达了目录f的结尾，返回值err会是io.EOF。如果n<=0，Readdir函数返回目录中剩余所有文件对象的名字构成的切片。此时，如果Readdir调用成功（读取所有内容直到结尾），它会返回该切片和nil的错误值。如果在到达结尾前遇到错误，会返回之前成功读取的名字构成的切片和该错误。
func (f *File) Truncate(size int64) error
//Truncate改变文件的大小，它不会改变I/O的当前位置。 如果截断文件，多出的部分就会被丢弃。如果出错，错误底层类型是*PathError。
func (f *File) Read(b []byte) (n int, err error)
//Read方法从f中读取最多len(b)字节数据并写入b。它返回读取的字节数和可能遇到的任何错误。文件终止标志是读取0个字节且返回值err为io.EOF。
func (f *File) ReadAt(b []byte, off int64) (n int, err error)
//ReadAt从指定的位置（相对于文件开始位置）读取len(b)字节数据并写入b。它返回读取的字节数和可能遇到的任何错误。当n<len(b)时，本方法总是会返回错误；如果是因为到达文件结尾，返回值err会是io.EOF。
func (f *File) Write(b []byte) (n int, err error)
//Write向文件中写入len(b)字节数据。它返回写入的字节数和可能遇到的任何错误。如果返回值n!=len(b)，本方法会返回一个非nil的错误。
func (f *File) WriteString(s string) (ret int, err error)
//WriteString类似Write，但接受一个字符串参数。
func (f *File) WriteAt(b []byte, off int64) (n int, err error)
//WriteAt在指定的位置（相对于文件开始位置）写入len(b)字节数据。它返回写入的字节数和可能遇到的任何错误。如果返回值n!=len(b)，本方法会返回一个非nil的错误。
func (f *File) Seek(offset int64, whence int) (ret int64, err error)
//Seek设置下一次读/写的位置。offset为相对偏移量，而whence决定相对位置：0为相对文件开头，1为相对当前位置，2为相对文件结尾。它返回新的偏移量（相对开头）和可能的错误。
func (f *File) Sync() (err error)
//Sync递交文件的当前内容进行稳定的存储。一般来说，这表示将文件系统的最近写入的数据在内存中的拷贝刷新到硬盘中稳定保存。
I/O operations
I/O operations are also called input and output operations. Among them, I refers to Input, and O refers to Output, which is used to read or write data. In some languages, it is also called stream operation, which refers to the channel of data communication.
The Golang standard library abstracts IO very delicately, and various components can be combined at will, which can be used as a model of interface design.
i包
The io package provides a series of interfaces for I/O primitive operations. It mainly wraps some existing implementations, such as those in the os package, and abstracts these into practical functions and some other related interfaces.
Since these interfaces and primitives wrap low-level operations in different implementations, clients should not assume that they are safe for parallel execution.
The two most important interfaces in the io package are the Reader and Writer interfaces. First, let's introduce these two interfaces.
Definition of the Reader interface, the Read() method is used to read data.
type Reader interface {
Read(p []byte) (n int, err error)
}
Read reads len§ bytes into p. It returns the number n of bytes read (0 <= n <= len§) and any errors encountered. Even if Read returns n < len§, it uses the entirety of p as scratch space during the call. If some data is available but less than len§ bytes, Read will return what is available as usual, rather than waiting for more.
When Read encounters an error or EOF condition after successfully reading n > 0 bytes, it returns the number of bytes read. It returns a (non-nil) error from the same call or an error (and n == 0) from subsequent calls. An example of this general case is a Reader that returns a non-zero byte count at the end of the input stream, either err == EOF or err == nil. In any case, the next Read should return 0, EOF.
The caller should always process bytes with n > 0 before considering err. Doing so correctly handles I/O errors after reading some bytes, and allows for EOF behavior.
The implementation of Read prevents returning a count of zero bytes and a nil error, which the caller should treat as a no-op.
Sample code:
package main
import (
"fmt"
"io"
"os"
)
func main() {
/*
读取数据：
Reader接口：
Read(p []byte)(n int, error)
*/
//读取本地1.txt文件中的数据
//step1：打开文件
fileName := "main/1.txt"
file, err := os.Open(fileName)
if err != nil {
fmt.Println("err:", err)
return
}
//step3：关闭文件
defer func(file *os.File) {
err := file.Close()
if err != nil {
}
}(file)
//step2：读取数据
bs := make([]byte, 4, 4)
/*
//第一次读取
n, err := file.Read(bs)
fmt.Println(err) //<nil>
fmt.Println(n) //4
fmt.Println(bs) //[97 98 99 100]
fmt.Println(string(bs)) //abcd
//第二次读取
n, err = file.Read(bs)
fmt.Println(err) //<nil>
fmt.Println(n) //4
fmt.Println(bs) //[101 102 103 104]
fmt.Println(string(bs)) //efgh
//第三次读取
n, err = file.Read(bs)
fmt.Println(err) //<nil>
fmt.Println(n) //2
fmt.Println(bs) //[105 106 103 104]
fmt.Println(string(bs)) //ijgh
//第四次读取
n, err = file.Read(bs)
fmt.Println(err) //EOF，文件的末尾
fmt.Println(n) //0
*/
n := -1
for {
n, err = file.Read(bs)
if n == 0 || err == io.EOF {
fmt.Println("读取到了文件的末尾，结束读取操作。。")
break
}
fmt.Println(n)
fmt.Println(string(bs[:n]))
}
/*
abcd
efgh
ij
读取到了文件的末尾，结束读取操作。。
*/
}
Definition of the Writer interface, the Write() method is used to write out data.
type Writer interface {
Write(p []byte) (n int, err error)
}
Write Writes len§ bytes from p to the elementary stream. It returns the number n of bytes written from p (0 <= n <= len§) and any errors encountered that caused the write to stop early. If Write returns n < len§, it must return a non-nil error. Write cannot modify the data of this slice, even if it is temporary.
Sample code:
package main
import (
"fmt"
"log"
"os"
)
func main() {
fileName := "main/1.txt"
//step1：打开文件
//step2：写出数据
//step3：关闭文件
//file, err := os.Open(fileName)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, os.ModePerm)
if err != nil {
fmt.Println(err)
return
}
defer file.Close()
//写出数据
//bs := []byte{65, 66, 67, 68, 69, 70} //A,B,C,D,E,F
//bs := []byte{97, 98, 99, 100} //a,b,c,d
//n,err := file.Write(bs)
//n, err := file.Write(bs[:2])
//fmt.Println(n)
//HandleErr(err)
//file.WriteString("\n")
//
//直接写出字符串
//n, err := file.WriteString("HelloWorld")
//fmt.Println(n)
//HandleErr(err)
file.WriteString("\n")
n, err := file.Write([]byte("today"))
fmt.Println(n)
HandleErr(err)
}
func HandleErr(err error) {
if err != nil {
log.Fatal(err)
}
}
The definition of the Seeker interface encapsulates the basic Seek method.
type Seeker interface {
Seek(offset int64, whence int) (int64, error)
}
The read and write pointer Seek used by Seeker to move data
sets the pointer position of the next read and write operation, and each read and write operation starts from the pointer position. The
meaning of whence:
if whence is 0: means to move the pointer from the beginning of the data
If whence is 1: means to move the pointer from the current pointer position of the data
If whence is 2: means to move the pointer from the end of the data
offset is the offset of the pointer movement
Returns the moved pointer position and any encountered during the movement mistake
The definition of the ReaderFrom interface encapsulates the basic ReadFrom method.
type ReaderFrom interface {
ReadFrom(r Reader) (n int64, err error)
}
ReadFrom reads data from r to the data stream of the object
until r returns EOF or r has a read error,
the return value n is the number of bytes read,
and the return value err is the return value err of r
The definition of the WriterTo interface encapsulates the basic WriteTo method.
type WriterTo interface {
WriteTo(w Writer) (n int64, err error)
}
WriterTo writes the data stream of the object into w
until the data stream of the object is completely written or encounters a write error.
The return value n is the number of bytes written,
and the return value err is the return value err of w
Define the ReaderAt interface, which encapsulates the basic ReadAt method
type ReaderAt interface {
ReadAt(p []byte, off int64) (n int, err error)
}
ReadAt Reads data from the off position of the object data stream into p.
Ignores the read and write pointers of the data, and starts reading from the offset off of the starting position of the data.
If the data stream of the object is only partially available and not enough to fill p,
then ReadAt It will wait for all the data to be available, then continue to write to p
until p is full and then return.
At this point, ReadAt is more strict than Read.
It returns the number of bytes read n and the error encountered when reading.
If n < len§, you need to return an err value to explain
why p is not filled (such as EOF)
if n = len§, and the object's data is not all read, then
err will return nil
if n = len§, and the object's If all the data has just been read,
err will return EOF or nil (not sure)
Define the WriterAt interface, which encapsulates the basic WriteAt method
type WriterAt interface {
WriteAt(p []byte, off int64) (n int, err error)
}
WriteAt Write the data in p to the off position of the object data stream
Ignore the read and write pointer of the data, and start writing from the offset off of the starting position of the data
Return the number of bytes written and the error encountered when writing
If n < len§, an err value must be returned stating
why p was not completely written
file copy
In the io package, there are mainly some methods for manipulating streams. Today, we will mainly learn about copy. Just copy a file to another directory.
Its principle is to read the data in the file from the source file through the program, and then write it out to the target file.
Read() and Write() under the io package
We can use the Read() and Write() methods under the io package to copy files while reading and writing. This method is to read the file in blocks, and the size of the block will also affect the performance of the program.
package main
import (
"fmt"
"io"
"os"
)
func main() {
srcFile := "main/1.txt"
destFile := "main/2.txt"
total, err := copyFile1(srcFile, destFile)
fmt.Println(total, err)
}
/*
该函数的功能：实现文件的拷贝，返回值是拷贝的总数量(字节),错误
*/
func copyFile1(srcFile, destFile string) (int, error) {
file1, err := os.Open(srcFile)
if err != nil {
return 0, err
}
file2, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
return 0, err
}
defer file1.Close()
defer file2.Close()
// 拷贝数据
bs := make([]byte, 1024, 1024)
n := -1 //读取的数据量
total := 0
for {
n, err = file1.Read(bs)
if err == io.EOF || n == 0 {
fmt.Println("拷贝完毕。。")
break
} else if err != nil {
fmt.Println("报错了。。。")
return total, err
}
total += n
file2.Write(bs[:n])
}
return total, nil
}
Copy() under the io package
We can also directly use the Copy() method under the io package.
The sample code is as follows:
package main
import (
"fmt"
"io"
"os"
)
func main() {
srcFile := "main/1.txt"
destFile := "main/2.txt"
total, err := copyFile2(srcFile, destFile)
fmt.Println(total, err)
}
func copyFile2(srcFile, destFile string) (int64, error) {
file1, err := os.Open(srcFile)
if err != nil {
return 0, err
}
file2, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
return 0, err
}
defer file1.Close()
defer file2.Close()
return io.Copy(file2, file1)
}
In the io package, not only the Copy() method is provided, but also two other public copy methods: CopyN() and CopyBuffer().
Copy（dst,src） //为复制src 全部到 dst 中。
CopyN(dst,src,n) //为复制src 中 n 个字节到 dst。
CopyBuffer（dst,src,buf）//为指定一个buf缓存区，以这个大小完全复制。
No matter which copy method is ultimately implemented by the private method copyBuffer().
func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {
// If the reader has a WriteTo method, use it to do the copy.
// Avoids an allocation and a copy.
if wt, ok := src.(WriterTo); ok {
return wt.WriteTo(dst)
}
// Similarly, if the writer has a ReadFrom method, use it to do the copy.
if rt, ok := dst.(ReaderFrom); ok {
return rt.ReadFrom(src)
}
if buf == nil {
size := 32 * 1024
if l, ok := src.(*LimitedReader); ok && int64(size) > l.N {
if l.N < 1 {
size = 1
} else {
size = int(l.N)
}
}
buf = make([]byte, size)
}
for {
nr, er := src.Read(buf)
if nr > 0 {
nw, ew := dst.Write(buf[0:nr])
if nw > 0 {
written += int64(nw)
}
if ew != nil {
err = ew
break
}
if nr != nw {
err = ErrShortWrite
break
}
}
if er != nil {
if er != EOF {
err = er
}
break
}
}
return written, err
}
As can be seen from this part of the code, there are three main types of replication.
1. If the copied Reader (src) will try to assert whether it can become a writerTo, if it can, directly call the following writerTo method
2. If Writer (dst) will try to assert whether it can be ReadFrom, if it can, directly call the following readfrom method
3. If there is no implementation, call the underlying read to realize the copy.
Among them, there is such a piece of code:
if buf == nil {
size := 32 * 1024
if l, ok := src.(*LimitedReader); ok && int64(size) > l.N {
if l.N < 1 {
size = 1
} else {
size = int(l.N)
}
}
buf = make([]byte, size)
}
This part mainly realizes the processing of Copy and CopyN. Through the above call diagram, we can see that CopyN will convert Reader into LimiteReader after calling.
The difference is that if Copy, directly create a buf with a default size of 32*1024 in the buffer area. If it is CopyN, it will first determine the number of bytes to be copied. If it is smaller than the default size, it will create a buf equal to the number of bytes to be copied.
ioutil package
ioutil.WriteFile()The third method is to use the sum in the ioutil package ioutil.ReadFile(), but because it uses the method of reading the file once and writing it again, this method is not suitable for large files and is prone to memory overflow.
Sample code:
package main
import (
"fmt"
"io/ioutil"
)
func main() {
srcFile := "main/1.txt"
destFile := "main/2.txt"
total, err := copyFile3(srcFile, destFile)
fmt.Println(total, err)
}
func copyFile3(srcFile, destFile string) (int, error) {
input, err := ioutil.ReadFile(srcFile)
if err != nil {
fmt.Println(err)
return 0, err
}
err = ioutil.WriteFile(destFile, input, 0644)
if err != nil {
fmt.Println("操作失败：", destFile)
fmt.Println(err)
return 0, err
}
return len(input), nil
}
Currently ReadFile and WriteFile are deprecated.
Summarize
Finally, let's test that these three types of copies take time. The copied files are all the same mp4 file (400M).
The first type: Read() and Write() under the io package directly read and write: the size of the slice we create to read data directly affects performance.
拷贝完毕。。
<nil>
401386819
real 0m7.911s
user 0m2.900s
sys 0m7.661s
The second method: the Copy() method under the io package:
<nil>
401386819
real 0m1.594s
user 0m0.533s
sys 0m1.136s
The third type: ioutil package
<nil>
401386819
real 0m1.515s
user 0m0.339s
sys 0m0.625s
These three methods, in terms of performance, whether it is io.Copy() or ioutil package, the performance is not bad.
http
Seeker interface
Seeker is an interface that wraps the basic Seek method.
type Seeker interface {
Seek(offset int64, whence int) (int64, error)
}
seek(offset,whence), set the position of the pointer cursor, read and write files randomly:
The first parameter: offset
The second parameter: how to set
0: seekStart means relative to the beginning of the file,
1: seekCurrent means relative to the current offset,
2: seek end means relative to the end.
1.txt content
ABCDEFababHelloWorldHelloWorld
Sample code:
package main
import (
"fmt"
"io"
"log"
"os"
)
func main() {
fileName := "main/1.txt"
file, err := os.OpenFile(fileName, os.O_RDWR, os.ModePerm)
if err != nil {
log.Fatal(err)
}
defer file.Close()
//读写
bs := []byte{
0}
file.Read(bs)
fmt.Println(string(bs))
file.Seek(4, io.SeekStart)
file.Read(bs)
fmt.Println(string(bs))
file.Seek(2, 0) //SeekStart
file.Read(bs)
fmt.Println(string(bs))
file.Seek(3, io.SeekCurrent)
file.Read(bs)
fmt.Println(string(bs))
file.Seek(0, io.SeekEnd)
file.WriteString("ABC")
}
operation result:
A
E
C
a
http
First think about a few questions
Q1: If the file you want to upload is relatively large, is there a way to shorten the time-consuming?
Q2: If the program is forced to be interrupted due to various reasons during the file transfer process, will the file need to be restarted next time when it is restarted?
Q3: When transferring files, does it support pause and resume? Even if these two operations are distributed before and after the program process is killed.
It can be realized by resuming transmission from breakpoints, and different languages have different implementation methods. Let's take a look at how to implement it through the Seek() method in Go language:
Let me talk about the idea first: If you want to resume the breakpoint transfer, the main thing is to remember how much data has been transferred last time, then we can create a temporary file to record the amount of data that has been transferred. When resuming the transfer, first from the temporary file Read the amount of data that has been transferred last time, and then set the read and write positions through the Seek() method, and then continue to transfer data.
Sample code:
package main
import (
"fmt"
"io"
"log"
"os"
"strconv"
"strings"
)
func main() {
srcFile := "main/1.txt"
destFile := srcFile[strings.LastIndex(srcFile, "/")+1:]
tempFile := destFile + "temp.txt"
file1, err := os.Open(srcFile)
file2, err := os.OpenFile(destFile, os.O_CREATE|os.O_WRONLY, os.ModePerm)
file3, err := os.OpenFile(tempFile, os.O_CREATE|os.O_RDWR, os.ModePerm)
defer file1.Close()
defer file2.Close()
//step1：先读取临时文件中的数据，再seek
file3.Seek(0, io.SeekStart)
bs := make([]byte, 100, 100)
n1, err := file3.Read(bs)
countStr := string(bs[:n1])
count, err := strconv.ParseInt(countStr, 10, 64)
//step2：设置读，写的位置：
file1.Seek(count, io.SeekStart)
file2.Seek(count, io.SeekStart)
data := make([]byte, 1024, 1024)
n2 := -1 //读取的数据量
n3 := -1 //写出的数据量
total := int(count) //读取的总量
//
//step3：复制文件
for {
n2, err = file1.Read(data)
if err == io.EOF || n2 == 0 {
fmt.Println("文件复制完毕。。")
file3.Close()
os.Remove(tempFile)
break
}
n3, err = file2.Write(data[:n2])
total += n3
//将复制的总量，存储到临时文件中
file3.Seek(0, io.SeekStart)
file3.WriteString(strconv.Itoa(total))
fmt.Printf("total:%d\n", total)
//假装断电
//if total > 8000{
// panic("假装断电了。。。")
//}
}
}
func HandleErr(err error) {
if err != nil {
log.Fatal(err)
}
}
bufio package
Principle of bufio package
bufio improves efficiency through buffering.
The efficiency of the io operation itself is not low, but the low frequency is the frequent access to files on the local disk. So bufio provides a buffer (allocate a block of memory), read and write in the buffer first, and finally read and write files to reduce the number of times to access the local disk, thereby improving efficiency.
To put it simply, when the file is read into the buffer (memory) and then read, the io of the file system can be avoided to improve the speed. Similarly, when performing a write operation, the file is first written to the buffer (memory), and then the buffer is written to the file system. After reading the above explanation, some people may be confused. Compared directly with content->file and content->buffer->file, the buffer does not seem to play a role. In fact, the buffer is designed to store multiple writes, and write the contents of the buffer to the file in one breath.
bufio wraps an io.Reader or io.Writer interface object and creates another object that also implements that interface.
The io.Reader or io.Writer interface implements the read() and write() methods, and these two methods can be used for objects that implement this interface.
Reader object
bufio.Reader is the encapsulation of io.Reader in bufio
// Reader implements buffering for an io.Reader object.
type Reader struct {
buf []byte
rd io.Reader // reader provided by the client
r, w int // buf read and write positions
err error
lastByte int // last byte read for UnreadByte; -1 means invalid
lastRuneSize int // size of last rune read for UnreadRune; -1 means invalid
}
bufio.Read(p []byte) is equivalent to reading the content of size len§, the idea is as follows:
- When the cache has content, fill all the contents of the cache into p and clear the cache
- When there is no content in the buffer area and len§>len(buf), that is, the content to be read is larger than the buffer area, just go to the file to read directly
- When the buffer has no content and len§<len(buf), that is, the content to be read is smaller than the buffer, the buffer reads the content from the file to fill the buffer, and fills up p (at this time, the buffer is left content)
- When reading again later, there is content in the buffer area, fill all the contents of the buffer area into p and clear the buffer area (this time is the same as case 1)
Sample code:
package main
import (
"bufio"
"fmt"
"os"
)
func main() {
fileName := "main/1.txt"
file, err := os.Open(fileName)
if err != nil {
fmt.Println(err)
return
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
}
}(file)
//创建Reader对象
//b1 := bufio.NewReader(file)
//1.Read()，高效读取
//p := make([]byte, 1024)
//n1, err := b1.Read(p)
//fmt.Println(n1)
//fmt.Println(string(p[:n1]))
//2.ReadLine()
//data, flag, err := b1.ReadLine()
//fmt.Println(flag)
//fmt.Println(err)
//fmt.Println(data)
//fmt.Println(string(data))
//3.ReadString()
//s1, err := b1.ReadString('\n')
//fmt.Println(err)
//fmt.Println(s1)
//s1, err = b1.ReadString('\n')
//fmt.Println(err)
//fmt.Println(s1)
//s1, err = b1.ReadString('\n')
//fmt.Println(err)
//fmt.Println(s1)
//
//for {
// s1, err := b1.ReadString('\n')
// if err == io.EOF {
// fmt.Println("读取完毕。。")
// break
// }
// fmt.Println(s1)
//}
//4.ReadBytes()
//data, err := b1.ReadBytes('\n')
//fmt.Println(err)
//fmt.Println(string(data))
//Scanner,输入的内容如果有空格，只能接收到空格前面的数据
//s2 := ""
//fmt.Scanln(&s2)
//fmt.Println(s2)
//可以接收到空格后面的数据
b2 := bufio.NewReader(os.Stdin)
s2, _ := b2.ReadString('\n')
fmt.Println(s2)
}
Writer object
bufio.Writer is the encapsulation of io.Writer in bufio
// Writer implements buffering for an io.Writer object.
// If an error occurs writing to a Writer, no more data will be
// accepted and all subsequent writes, and Flush, will return the error.
// After all data has been written, the client should call the
// Flush method to guarantee all data has been forwarded to
// the underlying io.Writer.
type Writer struct {
err error
buf []byte
n int
wr io.Writer
}
The idea of bufio.Write(p []byte) is as follows
- Determine whether the available capacity in buf can be placed p
- If it can be put down, directly splice p to the back of buf, that is, put the content in the buffer
- If the available capacity of the buffer is not enough to put it down, and the buffer is empty at this time, just write p to the file directly
- If the available capacity of the buffer is not enough to put it down, and the buffer has content at this time, use p to fill the buffer, write all the contents of the buffer to the file, and clear the buffer
- Determine whether the size of the remaining content of p can be put into the buffer, if it can be put down (this time is the same as step 1), then put the content into the buffer
- If the remaining content of p is still larger than the buffer (note that the buffer is empty at this time, the situation is the same as step 3), write the remaining content of p directly to the file
Sample code:
package main
import (
"bufio"
"fmt"
"os"
)
func main() {
fileName := "main/2.txt"
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
fmt.Println(err)
return
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
}
}(file)
w1 := bufio.NewWriter(file)
//n, err := w1.WriteString("helloworld")
//fmt.Println(err)
//fmt.Println(n)
//err = w1.Flush()
//if err != nil {
// return
//} //刷新缓冲区
for i := 1; i <= 1000; i++ {
_, err2 := w1.WriteString(fmt.Sprintf("%d:hello", i))
if err2 != nil {
return
}
}
err = w1.Flush()
if err != nil {
return
}
}
bufio package
The bufio package implements buffered I/O. It wraps an io.Reader or io.Writer interface object, creating another object that also implements the interface, and also provides buffering and some text I/O helper functions.
bufio.Reader
bufio.Reader implements the following interfaces:
io.Reader
io.WriterTo
io.ByteScanner
io.RuneScanner
// NewReaderSize 将 rd 封装成一个带缓存的 bufio.Reader 对象，
// 缓存大小由 size 指定（如果小于 16 则会被设置为 16）。
// 如果 rd 的基类型就是有足够缓存的 bufio.Reader 类型，则直接将
// rd 转换为基类型返回。
func NewReaderSize(rd io.Reader, size int) *Reader
// NewReader 相当于 NewReaderSize(rd, 4096)
func NewReader(rd io.Reader) *Reader
// Peek 返回缓存的一个切片，该切片引用缓存中前 n 个字节的数据，
// 该操作不会将数据读出，只是引用，引用的数据在下一次读取操作之
// 前是有效的。如果切片长度小于 n，则返回一个错误信息说明原因。
// 如果 n 大于缓存的总大小，则返回 ErrBufferFull。
func (b *Reader) Peek(n int) ([]byte, error)
// Read 从 b 中读出数据到 p 中，返回读出的字节数和遇到的错误。
// 如果缓存不为空，则只能读出缓存中的数据，不会从底层 io.Reader
// 中提取数据，如果缓存为空，则：
// 1、len(p) >= 缓存大小，则跳过缓存，直接从底层 io.Reader 中读
// 出到 p 中。
// 2、len(p) < 缓存大小，则先将数据从底层 io.Reader 中读取到缓存
// 中，再从缓存读取到 p 中。
func (b *Reader) Read(p []byte) (n int, err error)
// Buffered 返回缓存中未读取的数据的长度。
func (b *Reader) Buffered() int
// ReadBytes 功能同 ReadSlice，只不过返回的是缓存的拷贝。
func (b *Reader) ReadBytes(delim byte) (line []byte, err error)
// ReadString 功能同 ReadBytes，只不过返回的是字符串。
func (b *Reader) ReadString(delim byte) (line string, err error)
bufio.Writer
bufio.Writer implements the following interfaces:
io.Writer
io.ReaderFrom
io.ByteWriter
// NewWriterSize 将 wr 封装成一个带缓存的 bufio.Writer 对象，
// 缓存大小由 size 指定（如果小于 4096 则会被设置为 4096）。
// 如果 wr 的基类型就是有足够缓存的 bufio.Writer 类型，则直接将
// wr 转换为基类型返回。
func NewWriterSize(wr io.Writer, size int) *Writer
// NewWriter 相当于 NewWriterSize(wr, 4096)
func NewWriter(wr io.Writer) *Writer
// WriteString 功能同 Write，只不过写入的是字符串
func (b *Writer) WriteString(s string) (int, error)
// WriteRune 向 b 写入 r 的 UTF-8 编码，返回 r 的编码长度。
func (b *Writer) WriteRune(r rune) (size int, err error)
// Flush 将缓存中的数据提交到底层的 io.Writer 中
func (b *Writer) Flush() error
// Available 返回缓存中未使用的空间的长度
func (b *Writer) Available() int
// Buffered 返回缓存中未提交的数据的长度
func (b *Writer) Buffered() int
// Reset 将 b 的底层 Writer 重新指定为 w，同时丢弃缓存中的所有数据，复位
// 所有标记和错误信息。相当于创建了一个新的 bufio.Writer。
func (b *Writer) Reset(w io.Writer)
ioutil package
In addition to the io package that can read and write data, the Go language also provides an auxiliary toolkit called ioutil. Although there are not many methods in it, they are all pretty easy to use.
import "io/ioutil"
The introduction of the package has only one sentence: Package ioutil implements some I/O utility functions.
Methods of the ioutil package
Let's take a look at the method inside:
// Discard 是一个 io.Writer 接口，调用它的 Write 方法将不做任何事情
// 并且始终成功返回。
var Discard io.Writer = devNull(0)
// ReadAll 读取 r 中的所有数据，返回读取的数据和遇到的错误。
// 如果读取成功，则 err 返回 nil，而不是 EOF，因为 ReadAll 定义为读取
// 所有数据，所以不会把 EOF 当做错误处理。
func ReadAll(r io.Reader) ([]byte, error)
// ReadFile 读取文件中的所有数据，返回读取的数据和遇到的错误。
// 如果读取成功，则 err 返回 nil，而不是 EOF
func ReadFile(filename string) ([]byte, error)
// WriteFile 向文件中写入数据，写入前会清空文件。
// 如果文件不存在，则会以指定的权限创建该文件。
// 返回遇到的错误。
func WriteFile(filename string, data []byte, perm os.FileMode) error
// ReadDir 读取指定目录中的所有目录和文件（不包括子目录）。
// 返回读取到的文件信息列表和遇到的错误，列表是经过排序的。
func ReadDir(dirname string) ([]os.FileInfo, error)
// NopCloser 将 r 包装为一个 ReadCloser 类型，但 Close 方法不做任何事情。
func NopCloser(r io.Reader) io.ReadCloser
// TempFile 在 dir 目录中创建一个以 prefix 为前缀的临时文件，并将其以读
// 写模式打开。返回创建的文件对象和遇到的错误。
// 如果 dir 为空，则在默认的临时目录中创建文件（参见 os.TempDir），多次
// 调用会创建不同的临时文件，调用者可以通过 f.Name() 获取文件的完整路径。
// 调用本函数所创建的临时文件，应该由调用者自己删除。
func TempFile(dir, prefix string) (f *os.File, err error)
// TempDir 功能同 TempFile，只不过创建的是目录，返回目录的完整路径。
func TempDir(dir, prefix string) (name string, err error)
sample code
package main
import (
"fmt"
"io/ioutil"
"os"
)
func main() {
/*
ioutil包：
ReadFile()
WriteFile()
ReadDir()
..
*/
//1.读取文件中的所有的数据
//fileName := "main/1.txt"
//data, err := ioutil.ReadFile(fileName)
fmt.Println(err)
fmt.Println(data)
//fmt.Println(string(data))
//2.写出数据(覆盖写)
//fileName := "main/1.txt"
//s1 := "床前明月光，地上鞋三双"
//err := ioutil.WriteFile(fileName, []byte(s1), os.ModePerm)
//fmt.Println(err)
//3.ReadAll()
//s2 := "王二狗和李小花是两个好朋友，Ruby就是我，也是他们的朋友"
//r1 := strings.NewReader(s2)
//data, err := ioutil.ReadAll(r1)
//fmt.Println(err)
//fmt.Println(data)
//fmt.Println(string(data))
//4.ReadDir()，读取一个目录下的子内容：子文件和子目录，但是只能读取一层
//dirName := "main"
//fileInfos, err := ioutil.ReadDir(dirName)
//if err != nil {
// fmt.Println(err)
// return
//}
//fmt.Println(len(fileInfos))
//for i := 0; i < len(fileInfos); i++ {
// //fmt.Printf("%T\n", fileInfos[i])
// fmt.Printf("第 %d 个：名称：%s，是否是目录：%t\n", i, fileInfos[i].Name(), fileInfos[i].IsDir())
//}
//5.临时目录和临时文件
dir, err := ioutil.TempDir("./", "Test")
if err != nil {
fmt.Println(err)
return
}
defer os.Remove(dir)
fmt.Println(dir)
file, err := ioutil.TempFile(dir, "Test")
if err != nil {
fmt.Println(err)
return
}
defer os.Remove(file.Name())
fmt.Println(file.Name())
}
traverse folders
Because there are subfolders under the folder, and the ReadDir() of the ioutil package can only get one layer of directories, so we need to design an algorithm to achieve it. The easiest way to implement it is to use recursion.
Sample code:
package main
import (
"fmt"
"io/ioutil"
)
func main() {
dirName := "C:\\Users\\19393\\Desktop\\操作系统"
readDir(dirName)
}
func readDir(dirName string) {
fileInfos, err := ioutil.ReadDir(dirName)
if err != nil {
return
}
for i := 0; i < len(fileInfos); i++ {
if fileInfos[i].IsDir() {
dirName = dirName + "\\" + fileInfos[i].Name()
readDir(dirName)
} else {
fmt.Printf("文件名:%s\n", fileInfos[i].Name())
}
}
}
This package is currently deprecated
Concurrency
Multitasking
What is "multitasking"? Simply put, the operating system can run multiple tasks at the same time. For example, you are using a browser to surf the Internet, listening to MP3 while using Word to catch up with homework. This is multitasking, and at least 3 tasks are running at the same time. There are still many tasks running quietly in the background at the same time, but they are not displayed on the desktop.
The speed of the CPU is too fast. . .
what is concurrency
Go is a concurrent language, not a parallel language. Before discussing how to do concurrent processing in Go, we must first understand what concurrency is and how it differs from parallelism. (Go is a concurrent language and not a parallel one. )
Concurrency Concurrency is the ability to handle many things at the same time.
As an example, suppose a person is running in the morning. During his morning run, his shoelaces came loose. Now the person stops running, ties his shoes, and starts running again. This is a classic example of concurrency. This is a person who can handle running and tying shoes at the same time, which is a person who can handle many things at the same time.
What is parallelism, and how is it different from concurrency?
Parallelism is doing many things at the same time. This might sound similar to concurrency, but it's actually different.
Let's understand it better with the same jogging example. In this case, let's say the person is jogging and listening to music on his phone. In this case, a person jogging and listening to music is doing many things at the same time. This is called parallelism.
Concurrency and Parallelism - A Technical Viewpoint .
Suppose we are writing a web browser. A web browser has various components. Two of them are the web page rendering area and the downloader that downloads files from the internet. Suppose we structured the browser's code in such a way that each component executes independently. When the browser is running on a single-core processor, the processor will context switch between the two components of the browser. It might download a file for a while, then it might switch to rendering the html of the web page requested by the user. This is called concurrency. Concurrent processes start at different points in time and their execution cycles overlap. In this case, downloading and rendering start at different points in time and their execution overlaps.
Suppose the same browser is running on a multi-core processor. In this case, the file download component and the HTML rendering component may be running on different cores at the same time. This is called parallelism .
Parallelism Parallelism does not always result in faster execution times. This is because components running in parallel may need to communicate with each other. For example, in our browser, when a file download is complete, it should be communicated to the user, such as using a popup. This communication occurs between the component responsible for downloading and the component responsible for rendering the user interface. This communication overhead is low in concurrent systems. This communication overhead is high when components run concurrently across multiple cores. So parallel programs don't always result in faster execution times!
process, thread, coroutine
process
A process is a dynamic execution process of a program in a data set. It can be simply understood as an "executing program", which is an independent unit of CPU resource allocation and scheduling.
A process generally consists of three parts: program, data set, and process control block. The program we write is used to describe what functions the process needs to complete and how to complete it; the data set is the resources that the program needs to use during execution; the process control block is used to record the external characteristics of the process, describe the process of execution changes, and the system It can be used to control and manage processes, and it is the only sign that the system perceives the existence of a process. The limitation of the process is that the overhead of creating, revoking and switching is relatively large.
thread
Thread is a concept developed after process. A thread is also called a lightweight process. It is a basic CPU execution unit and the smallest unit in the program execution process. It consists of a thread ID, a program counter, a register set, and a stack. A process can contain multiple threads.
The advantage of threads is that it reduces the overhead of concurrent execution of programs and improves the concurrent performance of the operating system. The disadvantage is that threads do not have their own system resources, but only resources that are essential at runtime, but threads of the same process can share The system resources owned by the process, if the process is compared to a workshop, then the thread is like a worker in the workshop. However, there is a lock mechanism for some exclusive resources, and improper handling may cause "deadlock".
coroutine
A coroutine is a lightweight thread in user mode, also known as a micro-thread, and its English name is Coroutine. The scheduling of a coroutine is completely controlled by the user. People usually compare coroutines and subroutines (functions).
A subroutine call is always an entry and returns once, and once it exits, the execution of the subroutine is completed.
Compared with traditional system-level threads and processes, the biggest advantage of coroutines is its "lightweight", which can easily create millions without causing system resource exhaustion, and threads and processes usually cannot exceed 10,000 at most of. This is why coroutines are also called lightweight threads.
Compared with multi-threading, the advantages of coroutines are as follows: the execution efficiency of coroutines is extremely high. Because subroutine switching is not thread switching, but controlled by the program itself, there is no overhead of thread switching. Compared with multi-threading, the more threads there are, the more obvious the performance advantage of coroutines is.
The implementation of concurrency in Go language relies on coroutines, Goroutine
Coroutine in Go language - Goroutine
Process (Process), thread (Thread), coroutine (Coroutine, also called lightweight thread)
-
Process
A process is a dynamic execution process of a program in a data set. It can be simply understood as an "executing program", which is an independent unit of CPU resource allocation and scheduling.
A process generally consists of three parts: program, data set, and process control block. The program we write is used to describe what functions the process needs to complete and how to complete it; the data set is the resources that the program needs to use during execution; the process control block is used to record the external characteristics of the process, describe the process of execution changes, and the system It can be used to control and manage processes, and it is the only sign that the system perceives the existence of a process. The limitation of the process is that the overhead of creating, revoking and switching is relatively large. -
Threads
Threads are a concept developed after processes. A thread is also called a lightweight process. It is a basic CPU execution unit and the smallest unit in the program execution process. It is composed of a thread ID, a program counter, a register set, and a stack. A process can contain multiple threads.
The advantage of threads is that it reduces the overhead of concurrent execution of programs and improves the concurrent performance of the operating system. The disadvantage is that threads do not have their own system resources, but only resources that are essential at runtime, but threads of the same process can share The system resources owned by the process, if the process is compared to a workshop, then the thread is like a worker in the workshop. However, there is a lock mechanism for some exclusive resources, and improper handling may cause "deadlock". -
Coroutine
Coroutine is a lightweight thread in user mode, also known as micro-thread, the English name is Coroutine, and the scheduling of coroutine is completely controlled by the user. People usually compare coroutines and subroutines (functions). A subroutine call is always an entry and returns once, and once it exits, the execution of the subroutine is completed.
Compared with traditional system-level threads and processes, the biggest advantage of coroutines is its "lightweight", which can easily create millions without causing system resource exhaustion, and threads and processes usually cannot exceed 10,000 at most of. This is why coroutines are also called lightweight threads.
The characteristic of coroutine is that it is executed by one thread. Compared with multi-threading, its advantage is reflected in: the execution efficiency of coroutine is extremely high. Because subroutine switching is not thread switching, but controlled by the program itself, there is no overhead of thread switching. Compared with multi-threading, the more threads there are, the more obvious the performance advantage of coroutines will be.
Goroutine
What is Goroutine
Goroutine is used in go to achieve concurrency concurrently.
Goroutine is a noun specific to the Go language. Different from Process, Thread, and Coroutine, the creators of the Go language feel that it is different from them, so they created Goroutine specifically.
Goroutines are functions or methods that run concurrently with other functions or methods. Goroutines can be thought of as lightweight threads. Compared with threads, the cost of creating Goroutine is very small, it is a piece of code, a function entry. And a stack allocated for it on the heap (the initial size is 4K, it will automatically grow and delete as the program executes). So it's very cheap, and Go applications can run thousands of Goroutines concurrently.
Advantages of Goroutines over threads.
- Goroutines are very cheap compared to threads. They are just a few kb of the stack size, the stack can grow and shrink according to the needs of the application, whereas in the case of threads the stack size has to be specified and is fixed
- Goroutines are multiplexed into fewer OS threads. There may be only one thread and thousands of Goroutines in a program. If any Goroutines in the thread are said to be waiting for user input, another OS thread is created and the remaining Goroutines are transferred to the new OS thread. All of this is handled by the runtime, and we as programmers abstract away from these complex details and get a clean API related to concurrent work.
- Channels are designed to prevent race conditions when using Goroutines to access shared memory. Channels can be thought of as conduits for Goroutines to communicate.
main goroutine
The goroutine that encapsulates the main function is called the main goroutine.
What the main goroutine does is not as simple as executing the main function. The first thing it needs to do is: set the maximum size of the stack space that each goroutine can apply for. The maximum size is 250MB on a 32-bit computer system and 1GB on a 64-bit computer system. If a goroutine's stack size is larger than this limit, then the runtime system will cause a stack overflow (stack overflow) runtime panic. Subsequently, the operation of this go program will also terminate.
After that, the main goroutine will carry out a series of initialization work, and the work involved is roughly as follows:
-
Create a special defer statement to do the necessary aftermath when the main goroutine exits. Because the main goroutine may also end abnormally
-
Start a goroutine dedicated to cleaning memory garbage in the background, and set the GC available flag
-
Execute the init function in the mian package
-
Execute the main function
After executing the main function, it also checks whether the main goroutine caused a runtime panic, and performs necessary processing. Finally, the main goroutine will end itself and the current process.
How to use Goroutines
Prepend the keyword go to a function or method call, and you will simultaneously run a new Goroutine.
Example code:
package main
import (
"fmt"
)
func hello() {
fmt.Println("Hello world goroutine")
}
func main() {
go hello()
fmt.Println("main function")
}
Running result: It may output a value main function.
Since the main thread and the new goroutine are executed concurrently, they are independent of each other in time, so the main thread is likely to end immediately after printing "main function", and the new goroutine may not have enough time to execute the statement fmt.Println("Hello world goroutine"), As a result, the statement is not executed and printed.
What happened to the Goroutine we started? We need to understand the rules of Goroutine
- When a new Goroutine starts, the Goroutine call returns immediately. Unlike functions, Go does not wait for Goroutines to finish executing. When a Goroutine is invoked, and any return value from the Goroutine is ignored, go immediately executes to the next line of code.
- The main Goroutine should execute for other Goroutines. If main's Goroutine terminates, the program will be terminated and other Goroutines will not run.
Modify the above code:
package main
import (
"fmt"
"time"
)
func hello() {
fmt.Println("Hello world goroutine")
}
func main() {
go hello()
time.Sleep(1 * time.Second)
fmt.Println("main function")
}
In the above program, we have called the Sleep method of the time package, which will sleep during execution. In this case, the main goroutine is used to sleep for 1 second. The call to go hello() now has enough time to execute before the main Goroutine terminates. This program first prints the Hello world goroutine, waits for 1 second, then prints main function.
Start multiple Goroutines
Sample code:
package main
import (
"fmt"
"time"
)
func numbers() {
for i := 1; i <= 5; i++ {
time.Sleep(250 * time.Millisecond)
fmt.Printf("%d ", i)
}
}
func alphabets() {
for i := 'a'; i <= 'e'; i++ {
time.Sleep(400 * time.Millisecond)
fmt.Printf("%c ", i)
}
}
func main() {
go numbers()
go alphabets()
time.Sleep(3000 * time.Millisecond)
fmt.Println("main terminated")
}
operation result:
1 a 2 3 b 4 c 5 d e main terminated
Timeline Analysis:
Concurrency Model of Go Language
One of the great advantages of the Go language over Java and others is that it is easy to write concurrent programs. The Go language has a built-in goroutine mechanism. Using goroutine can quickly develop concurrent programs and make better use of multi-core processor resources. Next, let's take a look at the concurrency principle of the Go language.
threading model
In modern operating systems, threads are the basic unit of processor scheduling and allocation, and processes are the basic unit of resource ownership. Each process is composed of private virtual address space, code, data and various other system resources. A thread is an execution unit within a process. Each process has at least one main execution thread, which is automatically created by the system without being actively created by the user. Users create other threads in the application as needed, and multiple threads run concurrently in the same process.
Let's start with threads. Regardless of the concurrency model at the language level, it must exist in the form of threads at the operating system level. The operating system can be divided into user space and kernel space according to different resource access rights; the kernel space mainly operates and accesses hardware resources such as CPU resources, I/O resources, memory resources, etc., and provides the most basic basic resources for upper-layer applications. , The user space is the fixed activity space of the upper application program. The user space cannot directly access resources, and must call the resources provided by the kernel space through "system calls", "library functions" or "Shell scripts".
Our current computer language can be regarded as a kind of "software" in a narrow sense. The so-called "threads" in them are often user-mode threads, which are still different from the kernel-mode threads (KSE for short) of the operating system itself.
The Go concurrency programming model is supported by the thread library provided by the operating system at the bottom layer, so we still have to start with the thread implementation model.
Threads can be thought of as flow of control within a process. A process will contain at least one thread, because at least one flow of control in it will run continuously. Therefore, the first thread of a process will be created as the process starts, and this thread is called the main thread of the process. Of course, a process can also contain multiple threads. These threads are all created by existing threads in the current process. The method of creation is to call the system call, more precisely, to call the
pthread create function. A process with multiple threads can execute multiple tasks concurrently, and even if one or some tasks are blocked, it will not affect the normal execution of other tasks, which can greatly improve the response time and throughput of the program. On the other hand, threads cannot exist independently of processes. Its life cycle cannot exceed the life cycle of the process to which it belongs.
There are three main thread implementation models, namely: user-level thread model, kernel-level thread model, and two-level thread model. The biggest difference between them lies in the corresponding relationship between threads and kernel scheduling entities (Kernel Scheduling Entity, KSE for short). As the name implies, a kernel scheduling entity is an object that can be scheduled by the kernel's scheduler. In many documents and books, it is also called a kernel-level thread, which is the smallest scheduling unit of the operating system kernel.
Kernel-Level Threading Model
There is a one-to-one relationship (1:1) between user threads and KSE. The thread library of most programming languages (such as pthread of linux, java.lang.Thread of Java, std::thread of C++11, etc.) Each created thread is statically associated with a different KSE, so its scheduling is completely done by the OS scheduler. This method is simple to implement, directly using the thread capabilities provided by the OS, and generally does not affect each other between different user threads. However, operations such as its creation, destruction, and context switching between multiple threads are all performed directly by the OS layer, which will have a great impact on the performance of the OS in scenarios where a large number of threads are required. Each thread is scheduled independently by the kernel scheduler, so if one thread blocks it does not affect other threads.
Advantages: With the support of multi-core processor hardware, the kernel space thread model supports true parallelism. When one thread is blocked, another thread is allowed to continue execution, so the concurrency capability is strong.
Disadvantages: Every time a user-level thread is created, a kernel-level thread needs to be created to correspond to it, so the overhead of creating a thread is relatively large, which will affect the performance of the application.
user-level threading model
The user thread and KSE have a many-to-one relationship (M:1). The creation, destruction, and coordination among multiple threads of this thread are all in charge of the thread library implemented by the user, which is transparent to the OS kernel. All threads created in a process are dynamically associated with the same KSE at runtime. Coroutines implemented in many languages now basically belong to this method. Compared with kernel-level threads, this implementation can be very lightweight, and consumes much less system resources, so the number of threads that can be created and the cost of context switching will be much smaller. But this model has a fatal shortcoming. If we call a blocking system call (such as reading network IO in a blocking way) on a user thread, then once KSE is scheduled out of the CPU by the kernel due to blocking, all remaining corresponding User threads all become blocked (the whole process hangs). Therefore, the coroutine libraries
of these languages will repackage some of their own blocking operations into a completely non-blocking form, and then actively give up at the point that was previously blocked, and notify or wake up other users to be executed in some way The thread runs on the KSE, thereby avoiding the context switch of the kernel scheduler due to KSE blocking, so that the entire process will not be blocked.
Advantages: The advantage of this model is that thread context switches all occur in user space, avoiding mode switches (mode switch), which has a positive impact on performance.
Disadvantages: All threads are based on a kernel scheduling entity, the kernel thread, which means that only one processor can be utilized, which is unacceptable in a multi-processor environment. In essence, user threads only solve the concurrency problem. But it doesn't solve the parallel problem. If the thread falls into the kernel mode due to I/O operation, and the kernel mode thread is blocked waiting for I/O data, all threads will be blocked, and user space can also use non-blocking I/O, but performance and complexity cannot be avoided. Degree problem.
two-level threading model
User threads and KSE have a many-to-many relationship (M:N). This implementation combines the advantages of the first two models to create multiple KSEs for one process, and threads can be dynamically associated with different KSEs at runtime. When a certain KSE is scheduled out of the CPU by the kernel due to the blocking operation of the working thread on it, the remaining user threads currently associated with it can re-establish associations with other KSEs. Of course, the implementation of this dynamic association mechanism is very complicated, and users need to implement it themselves. This is one of its shortcomings. Concurrency in the Go language is the implementation method used. In order to realize this model, Go implements a runtime scheduler to be responsible for the dynamic association between "threads" in Go and KSE. This model is sometimes called a hybrid threading model , that is, the user scheduler realizes the "scheduling" of user threads to KSE, and the kernel scheduler realizes the scheduling of KSE to CPU .
Go Concurrency Scheduling: GPM Model
Go builds a unique two-level threading model on top of the kernel threads provided by the operating system. The goroutine mechanism implements the M:N thread model. The goroutine mechanism is an implementation of a coroutine. The built-in scheduler of golang allows each CPU in a multi-core CPU to execute a coroutine.
How the scheduler works
With the above understanding, we can start to really introduce the concurrency mechanism of Go. First, we will use a piece of code to show how to create a new "thread" (called Goroutine in Go language) in Go language:
// 用go关键字加上一个函数（这里用了匿名函数）
// 调用就做到了在一个新的“线程”并发执行任务
go func() {
// do something in one new goroutine
}()
The key to understanding the principle of the goroutine mechanism is to understand the implementation of the Go language scheduler.
There are four important structures supporting the implementation of the entire scheduler in the Go language, namely M, G, P, and Sched. The first three are defined in runtime.h, and Sched is defined in proc.c.
- The Sched structure is the scheduler, which maintains the queues storing M and G and some status information of the scheduler.
- The M structure is Machine, a system thread, which is managed by the operating system, and goroutine runs on top of M; M is a large structure, which maintains a small object memory cache (mcache), the currently executing goroutine, and random number generation device and so on a lot of information.
- P structure is Processor, processor, its main purpose is to execute goroutine, it maintains a goroutine queue, namely runqueue. Processor is an important part that allows us to schedule from N:1 to M:N.
- G is the core structure of goroutine implementation, which contains stack, instruction pointer, and other information important for scheduling goroutine, such as its blocked channel.
The number of Processors is set as the value of the environment variable GOMAXPROCS at startup, or by calling the function GOMAXPROCS() at runtime. A fixed number of Processors means that only GOMAXPROCS threads are running go code at any one time.
We represent Machine Processor and Goroutine with triangle, rectangle and circle respectively.
In the scenario of a single-core processor, all goroutines run in the same M system thread, and each M system thread maintains a Processor. At any time, there is only one goroutine in a Processor, and other goroutines are waiting in the runqueue. After a goroutine finishes running its own time slice, it gives up the context and returns to the runqueue. In the scenario of multi-core processors, in order to run goroutines, each M system thread will hold a Processor.
Under normal circumstances, the scheduler will schedule according to the above process, but the thread will be blocked, etc., look at how goroutine handles thread blocking.
thread blocking
When the running goroutine is blocked, such as making a system call, another system thread (M1) will be created, the current M thread gives up its Processor, and P switches to a new thread to run.
runqueue execution completed
When one of the Processor's runqueue is empty, no goroutine can be scheduled. It will steal half of the goroutines from another context.
G, P and M in the figure are the Go language runtime system (including memory allocator, concurrent scheduler, garbage collector and other components, which can be imagined as the JVM in Java) to abstract concepts and data structure objects:
G : The abbreviation of Goroutine, the above code using the go keyword plus function call creates a G object, which is the encapsulation of a task to be executed concurrently, and can also be called a user mode thread. It is a user-level resource, transparent to the OS, lightweight, can be created in large quantities, and has the characteristics of low context switching cost.
M: The abbreviation of Machine, which is created by the clone system call on the linux platform, which is essentially the same as the thread created by the linux pthread library, and is an OS thread entity created by the system call. The role of M is to execute the concurrent tasks packaged in G. The main responsibility of the scheduler in the Go runtime system is to arrange G to be executed on multiple M in a fair and reasonable manner . It is an OS resource, and the number that can be created is also limited by the OS. Usually, the number of Gs is more than that of active Ms.
P: short for Processor, logical processor, the main function is to manage G objects (each P has a G queue), and provide localized resources for the operation of G on M.
From the perspective of the two-level thread model, it seems that the participation of P is not required, and G and M are enough, so why add P?
In fact, the early implementation of the Go language runtime system (Go1.0) does not have the concept of P. The scheduler in Go directly assigns G to the appropriate M to run. But this brings many problems. For example, when different Gs run concurrently on different Ms, they may all need to apply for resources (such as heap memory) from the system. Since the resources are global, a lot of system performance loss will be caused by resource competition. , in order to solve similar problems, the later Go (Go1.1) runtime system added P, allowing P to manage G objects. If M wants to run G, it must first be bound to a P, and then the G managed by the P can be run. . The advantage of this is that we can pre-apply for some system resources (local resources) in the P object. When G needs it, it first applies to its own local P (without lock protection). If it is not enough or does not apply to the global, And when you take it from the overall situation, you will take an extra part for efficient use later. Just like when we go to the government to do things now, we first go to the local government to see if it can be done, and if we can’t, then go to the central government, so as to improve work efficiency.
And because P decouples G and M objects, even if M is blocked by the running G on it, the remaining G associated with the M can migrate to other active Ms along with P to continue running, thus Let G always find M in time and run itself, thereby improving the concurrency capability of the system.
The Go runtime system implements a user-mode concurrent scheduling system by constructing the GPM object model, which can manage and schedule its own concurrent tasks, so it can be said that the Go language natively supports concurrency . The self-implemented scheduler is responsible for assigning concurrent tasks to different kernel threads to run, and then the kernel scheduler takes over the execution and scheduling of kernel threads on the CPU.
at last
The complete scheduling system of the Go runtime is very complicated, and it is difficult to describe it clearly in an article. Here we can only introduce it from a macro perspective, so that everyone can have an overall understanding.
// Goroutine1
func task1() {
go task2()
go task3()
}
If we have a G (Goroutine1) that has been scheduled to be executed on an M through P, and we create two Gs during the execution of Goroutine1, these two Gs will be immediately placed in the local G of the same P as Goroutine1 In the task queue, wait for the execution of the M bound to the P. This is the most basic structure and it is easy to understand. The key questions are:
a. How to reasonably allocate G to multiple M on a multi-core system, make full use of multi-core, and improve concurrency capabilities? If we create a large number of Gs in a Goroutine through the go keyword, these Gs will be placed in the same queue temporarily, but if there are idle Ps at this time (the number of Ps in the system is equal to the number of system cpu cores by default)
, Go The runtime system can always guarantee that there is at least one (usually only one) active M and idle P bound to various G queues to find runnable G tasks. This kind of M is called spinning M. The general search order is: the queue of the P that is bound to itself, the global queue, and then other P queues. If you find it in your P queue, take it out and start running. Otherwise, go to the global queue to see. Since the global queue needs lock protection, if there are many tasks in it, a batch will be transferred to the local P queue to avoid competing for locks every time. If there is still no global queue, it is time to start playing hard, and directly steal tasks from other P queues (steal half of the tasks back). In this way, it is ensured that there are always M+P combinations equal to the number of CPU cores executing G tasks or on the way to execute G (looking for G tasks) when there are still executable G tasks.
b. What if a certain M is blocked by a system call in G during the execution of G?
In this case, this M will be scheduled out of the CPU by the kernel scheduler and will be in a blocked state, and other Gs associated with this M will have no way to continue executing, but a monitoring thread (sysmon thread) of the Go runtime system can Such an M is detected, and the P bound to the M is stripped off, another idle or new M is found to take over the P, and then the G among them continues to run. The general process is shown in the figure below. Then when the M recovers from the blocked state, it is necessary to find another idle P to continue executing the original G. If the system happens to have no idle P at this time, put the original G into the global queue and wait for other M+P combinations to be discovered. and execute.
c. If a certain G runs for too long in M, is there a way to do preemptive scheduling, so that other Gs on the M can get a certain amount of running time to ensure the fairness of the scheduling system? We know that the kernel scheduler of Linux
mainly Scheduling is based on time slices and priorities. For threads of the same priority, the kernel scheduler will try to ensure that each thread can obtain a certain execution time. In order to prevent some threads from "starving to death", the kernel scheduler will initiate preemptive scheduling to interrupt long-running threads and give up CPU resources, allowing other threads to get execution opportunities. Of course, there is a similar preemption mechanism in the Go runtime scheduler, but there is no guarantee that the preemption will succeed, because the Go runtime system does not have the interrupt capability of the kernel scheduler, it can only set The method of preempting the flag gently allows the running G to voluntarily give up the execution right of M.
Having said that, I have to mention the ability of Goroutine to dynamically expand its own thread stack during operation. It can expand from the initial 2KB to a maximum of 1G (on a 64bit system). Therefore, it is necessary to calculate the function before calling the function each time. The size of the stack space required by the call, and then expanded as needed (exceeding the maximum will cause a runtime exception). The mechanism of Go preemptive scheduling is to check the following preemptive flags by the way when judging whether to expand the stack, and decide whether to continue execution or give up.
The monitoring thread of the runtime system will time and set the preemption flag to the G that is running for too long, and then G will check the preemption flag when there is a function call. If it has been set, it will put itself into the global queue, so that the associated on the M Other G's have the opportunity to execute. But if the G being executed is a very time-consuming operation and there is no function call (such as a calculation operation in a for loop), even if the preemption flag has been set, the G will still occupy the current M until its own task is executed. .
example
package main
import (
"fmt"
"time"
)
func main() {
/*
一个goroutine打印数字，另外一个goroutine打印字母，观察运行结果。。
并发的程序的运行结果，每次都不一定相同。
不同计算机设备执行，效果也不相同。
go语言的并发：go关键字
系统自动创建并启动主goroutine，执行对应的main()
用于自己创建并启动子goroutine，执行对应的函数
go 函数()//go关键创建并启动goroutine，然后执行对应的函数()，该函数执行结束，子goroutine也随之结束。
子goroutine中执行的函数，往往没有返回值。
如果有也会被舍弃。
*/
//1.先创建并启动子goroutine，执行printNum()
go printNum()
//2.main中打印字母
for i := 1; i <= 10; i++ {
fmt.Printf("\t主goroutine中打印字母：A %d\n", i)
}
time.Sleep(1 * time.Second)
fmt.Println("main...over...")
}
func printNum() {
for i := 1; i <= 10; i++ {
fmt.Printf("子goroutine中打印数字：%d\n", i)
}
}
operation result:
主goroutine中打印字母：A 1
子goroutine中打印数字：1
子goroutine中打印数字：2
子goroutine中打印数字：3
子goroutine中打印数字：4
子goroutine中打印数字：5
子goroutine中打印数字：6
子goroutine中打印数字：7
主goroutine中打印字母：A 2
主goroutine中打印字母：A 3
主goroutine中打印字母：A 4
主goroutine中打印字母：A 5
主goroutine中打印字母：A 6
主goroutine中打印字母：A 7
主goroutine中打印字母：A 8
子goroutine中打印数字：8
子goroutine中打印数字：9
主goroutine中打印字母：A 9
子goroutine中打印数字：10
主goroutine中打印字母：A 10
main...over...
runtime package
Although the Go compiler produces native executable code, the code still runs in the Go runtime (this part of the code can be found in the runtime package). This runtime is similar to the virtual machine used by Java and .NET languages. It is responsible for management including memory allocation, garbage collection, stack processing, goroutine, channel, slice (slice), map and reflection (reflection) and so on.
Common functions
runtimeThe scheduler is a very useful thing, and there runtimeare several methods about the package:
-
NumCPU : returns
CPUthe number of cores in the current system -
GOMAXPROCS
CPU: Set the maximum number of cores that can be used simultaneouslyThrough the runtime.GOMAXPROCS function, how does the application set the maximum number of P in the runtime system during runtime. But this causes "Stop the World". Therefore, it should be called at the earliest in the application. And it is better to set the environment variable GOMAXPROCS of the operating program before running the Go program, instead of calling the runtime.GOMAXPROCS function in the program.
No matter what integer value we pass to the function, the maximum value of P for the runtime system will always be between 1 and 256.
After go1.8, the program runs on multiple cores by default. You don’t need to set it
before go1.8. You still need to set it up to benefit the CPU more efficiently.
-
Gosched : Let the current thread give up
cputo let other threads run, it will not suspend the current thread, so the current thread will continue to execute in the futureThe function of this function is to let the current thread
goroutinegive upCPU. When agoroutinethread is blocked,Goit will automatically transfergoroutineother threads that are in the same system thread as thegoroutinethread to another system thread, so that these threadsgoroutinewill not block. -
Goexit : exit the current
goroutine(butdeferthe statement will be executed as usual) -
NumGoroutine : Returns the total number of executing and queued tasks
The runtime.NumGoroutine function returns the number of Goroutines in a particular state in the system after being called. The specific reference here refers to Grunnable\Gruning\Gsyscall\Gwaition. Grooutines in these states are considered active or being scheduled.
Note: If the state of the Grooutine where the garbage collection is located is also within this range, it will also be included in this counter.
-
GOOS : target operating system
-
runtime.GC : will cause the runtime system to perform a mandatory garbage collection
- Mandatory Garbage Collection: Garbage collection that must be performed no matter what.
- Non-mandatory garbage collection: Garbage collection will only be performed under certain conditions (that is, at runtime, the newly-applied heap memory unit (also called unit increment) of the system since the last garbage collection reaches the specified value).
-
GOROOT : Get the goroot directory
-
GOOS : Check the target operating system
Many times, we will implement different operations according to different platforms, so we just use GOOS:
sample code
- Get goroot and os:
//获取goroot目录：
fmt.Println("GOROOT-->",runtime.GOROOT())
//获取操作系统
fmt.Println("os/platform-->",runtime.GOOS)
GOROOT--> C:\Users\19393\sdk\go1.20.4
os/platform--> windows
- Get the number of CPUs, and set the number of CPUs:
func init(){
//1.获取逻辑cpu的数量
fmt.Println("逻辑CPU的核数：",runtime.NumCPU()) //16
//2.设置go程序执行的最大的：[1,256]
n := runtime.GOMAXPROCS(runtime.NumCPU())
fmt.Println(n) //16
}
- Gosched()：
func main() {
go func() {
for i := 0; i < 5; i++ {
fmt.Println("goroutine")
}
}()
for i := 0; i < 4; i++ {
//让出时间片，先让别的协议执行，它执行完，再回来执行此协程
runtime.Gosched()
fmt.Println("main")
}
}
goroutine
goroutine
goroutine
goroutine
goroutine
main
main
main
main
- Use of Goexit (terminate coroutine)
package main
import (
"fmt"
"runtime"
"time"
)
func main() {
//创建新建的协程
go func() {
fmt.Println("goroutine开始")
//调用了别的函数
fun()
fmt.Println("goroutine结束")
}() //别忘了()
//睡一会儿，不让主协程结束
time.Sleep(3 * time.Second)
}
func fun() {
defer fmt.Println("defer")
//return //终止此函数
runtime.Goexit() //终止所在的协程
fmt.Println("fun函数")
}
goroutine开始
defer
critical resource
Critical resources: Refers to resources shared by multiple processes/threads/coroutines in a concurrent environment.
However, improper handling of critical resources in concurrent programming often leads to data inconsistency.
Sample code:
package main
import (
"fmt"
"time"
)
func main() {
a := 1
go func() {
a = 2
fmt.Println("子goroutine",a)
}()
a = 3
time.Sleep(1)
fmt.Println("main goroutine",a)
}
子goroutine 2
main goroutine 2
A data a shared by multiple goroutines can be found.
Critical Resource Security Issues
Concurrency itself is not complicated, but because of the problem of resource competition, it makes it complicated for us to develop good concurrent programs, because it will cause many inexplicable problems.
If one of the threads modifies the data when multiple goroutines are accessing the same data resource, then this value will be modified. For other goroutines, this value may be wrong.
For example, we implement the train station ticketing program through concurrency. There are 10 tickets in total, and 4 ticket outlets are sold at the same time.
Let's take a look at the sample code first:
package main
import (
"fmt"
"math/rand"
"time"
)
// 全局变量
var ticket = 10 //
func main() {
/*
4个goroutine，模拟4个售票口，4个子程序操作同一个共享数据。
*/
go saleTickets("售票口1") // g1,10
go saleTickets("售票口2") // g2,10
go saleTickets("售票口3") //g3,10
go saleTickets("售票口4") //g4,10
time.Sleep(5 * time.Second)
}
func saleTickets(name string) {
rand.Seed(time.Now().UnixNano())
for {
//ticket=1
if ticket > 0 {
//g1,g3,g2,g4
//睡眠
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
// g1 ,g3, g2,g4
fmt.Println(name, "售出：", ticket)
ticket--
} else {
fmt.Println(name, "售罄，没有票了。。")
break
}
}
}
operation result:
售票口4 售出： 10
售票口1 售出： 10
售票口3 售出： 10
售票口2 售出： 10
售票口1 售出： 6
售票口1 售出： 5
售票口2 售出： 4
售票口4 售出： 3
售票口4 售出： 2
售票口1 售出： 1
售票口1 售罄，没有票了。。
售票口3 售出： 0
售票口3 售罄，没有票了。。
售票口4 售出： -1
售票口4 售罄，没有票了。。
售票口2 售出： -2
售票口2 售罄，没有票了。。
In order to better observe the problem of critical resources, each goroutine sleeps with a random number first, and then sells tickets. We found that the running results of the program can also sell tickets with negative numbers.
analyze:
Our ticket selling logic is to first judge whether the ticket number is negative. If it is greater than 0, then we will sell the ticket, but we will sleep before selling the ticket, and then sell it again. If the ticket has been sold to the last 1, a certain goroutine holds the time slice of the CPU, then when it has a ticket in the segment, the condition is true, so it can sell the last ticket with ticket number 1. But because it sleeps before selling, then other goroutines will hold the time slice of the CPU, and this ticket has not been sold at this time, then when the second goroutine judges whether there is a ticket, the condition is also established, then it can sell the ticket, but it also fell asleep. The other third and fourth goroutines all have this logic. When a certain goroutine wakes up, it will no longer judge whether there is a ticket, but sell it directly, so that the last ticket is sold. However, other When the goroutine wakes up, it will sell the 0th, -1, and -2 one after another.
This is the insecurity of critical resources. When a certain goroutine accesses a data resource, it has judged the condition according to the value, and then the resource is seized by other goroutines, and the value is modified. When the goroutine continues to access the data, the value is no longer correct. up.
The Solution to Critical Resource Safety Problem
To solve the problem of critical resource safety, the solution of many programming languages is synchronization. By locking, only one goroutine is allowed to access the shared data in a certain period of time. After the current goroutine has finished accessing, other goroutines can access it after unlocking.
We can use the lock operation under the sync package.
Sample code:
package main
import (
"fmt"
"math/rand"
"time"
"sync"
)
//全局变量
var ticket = 10 // 10张票
var wg sync.WaitGroup
var matex sync.Mutex // 创建锁头
func main() {
/*
4个goroutine，模拟4个售票口，4个子程序操作同一个共享数据。
*/
wg.Add(4)
go saleTickets("售票口1") // g1,100
go saleTickets("售票口2") // g2,100
go saleTickets("售票口3") //g3,100
go saleTickets("售票口4") //g4,100
wg.Wait() // main要等待。。。
//time.Sleep(5*time.Second)
}
func saleTickets(name string) {
rand.Seed(time.Now().UnixNano())
defer wg.Done()
//for i:=1;i<=100;i++{
// fmt.Println(name,"售出：",i)
//}
for {
//ticket=1
matex.Lock()
if ticket > 0 {
//g1,g3,g2,g4
//睡眠
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
// g1 ,g3, g2,g4
fmt.Println(name, "售出：", ticket) // 1 , 0, -1 , -2
ticket-- //0 , -1 ,-2 , -3
} else {
matex.Unlock() //解锁
fmt.Println(name, "售罄，没有票了。。")
break
}
matex.Unlock() //解锁
}
}
operation result:
售票口1 售出： 10
售票口1 售出： 9
售票口4 售出： 8
售票口3 售出： 7
售票口2 售出： 6
售票口1 售出： 5
售票口4 售出： 4
售票口3 售出： 3
售票口2 售出： 2
售票口1 售出： 1
售票口4 售罄，没有票了。。
售票口1 售罄，没有票了。。
售票口3 售罄，没有票了。。
售票口2 售罄，没有票了。。
at last
There is a classic saying in Go's concurrent programming: don't communicate by sharing memory, but share memory by communicating.
In the Go language, it is not encouraged to use locks to protect the shared state to share information in different Goroutines (communicate in the form of shared memory). Instead, it is encouraged to pass the shared state or the change of the shared state between Goroutines through the channel (to share memory by means of communication), so that it can also ensure that only one Goroutine accesses the shared state at the same time like using a lock.
Of course, in order to ensure the security and consistency of shared data among multiple threads, mainstream programming languages will provide a set of basic synchronization tools, such as locks, condition variables, atomic operations, and so on. It is no surprise that the Go language standard library also provides these synchronization mechanisms, and the usage is similar to other languages.
WaitGroup
Sync is the abbreviation of the word synchronization, so it is also called a synchronization package. Basic synchronization operations are provided here, such as mutexes and so on. Here, except for the Once and WaitGroup types, most of the types are used by the low-level library routines. Higher levels of synchronization are best accomplished through channels and communication.
WaitGroup, synchronous waiting group.
Typewise, it's a struct. The purpose of a WaitGroup is to wait for a collection of goroutines to complete. The main goroutine calls the Add() method to set the number of goroutines to wait. Then, each goroutine will execute and call the Done() method after execution. At the same time, you can use the Wait() method to block until all goroutines are executed.
Add() method
The Add method is used to set the value of the counter to the WaitGroup. We can understand that there is a counter in each waitgroup
to indicate the number of goroutins to be executed in this synchronous waiting group.
If the value of the counter becomes 0, it means that all goroutines blocked while waiting are released. If the value of the counter is negative, a panic will be triggered and the program will report an error.
Done() method
The Done() method is to set the counter value of the WaitGroup to decrease by 1 after a certain goroutine in the WaitGroup synchronously waits for the group to execute.
In fact, the underlying code of Done() is to call the Add() method:
// Done decrements the WaitGroup counter by one.
func (wg *WaitGroup) Done() {
wg.Add(-1)
}
Wait () method
The Wait() method means to let the current goroutine wait and enter the blocking state. Until the WaitGroup's counter is zero. In order to unblock,
this goroutine can continue to execute.
sample code
We create and start two goroutines to print numbers and letters, and in the main goroutine, add these two sub-goroutines to a WaitGroup, and let the main goroutine enter Wait(), and let the two sub-goroutines execute first. When each child goroutine is executed, the Done() method is called, and the counter of the WaitGroup is decremented by 1. When both sub-goroutines are executed, the value of the counter in the WaitGroup is zero, unblocking the main goroutine.
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup // 创建同步等待组对象
func main() {
/*
WaitGroup：同步等待组
可以使用Add(),设置等待组中要 执行的子goroutine的数量，
在main 函数中，使用wait(),让主程序处于等待状态。直到等待组中子程序执行完毕。解除阻塞
子gorotuine对应的函数中。wg.Done()，用于让等待组中的子程序的数量减1
*/
//设置等待组中，要执行的goroutine的数量
wg.Add(2)
go fun1()
go fun2()
fmt.Println("main进入阻塞状态。。。等待wg中的子goroutine结束。。")
wg.Wait() //表示main goroutine进入等待，意味着阻塞
fmt.Println("main，解除阻塞。。")
}
func fun1() {
for i := 1; i <= 10; i++ {
fmt.Println("fun1.。。i:", i)
}
wg.Done() //给wg等待中的执行的goroutine数量减1.同Add(-1)
}
func fun2() {
defer wg.Done()
for j := 1; j <= 10; j++ {
fmt.Println("\tfun2..j,", j)
}
}
operation result:
main进入阻塞状态。。。等待wg中的子goroutine结束。。
fun2..j, 1
fun2..j, 2
fun2..j, 3
fun2..j, 4
fun2..j, 5
fun2..j, 6
fun2..j, 7
fun2..j, 8
fun2..j, 9
fun2..j, 10
fun1.。。i: 1
fun1.。。i: 2
fun1.。。i: 3
fun1.。。i: 4
fun1.。。i: 5
fun1.。。i: 6
fun1.。。i: 7
fun1.。。i: 8
fun1.。。i: 9
fun1.。。i: 10
main，解除阻塞。。
mutex
Mutex (mutual exclusion lock)
In concurrent programs, there will be critical resource problems. That is, when multiple coroutines access shared data resources, then this shared resource is not safe. In order to solve the problem of coroutine synchronization, we use channel, but Go language also provides traditional synchronization tools.
What is a lock? That is, a certain coroutine (thread) locks first when accessing a certain resource to prevent access by other coroutines. After the access is completed and unlocked, other coroutines lock it for access. It is generally used to deal with critical resource issues in concurrency.
The sync package in the Go language package provides two lock types: sync.Mutex and sync.RWMutex.
Mutex is the simplest type of lock. Mutex is also relatively violent. When a goroutine acquires the Mutex, other goroutines can only obediently wait until the goroutine releases the Mutex.
Each resource corresponds to a mark that can be called a "mutual exclusion lock", which is used to ensure that only one coroutine (thread) can access the resource at any time. Other coroutines can only wait.
Mutex is the main means of traditional concurrent programming to control access to shared resources, and it is represented by the Mutex structure type in the standard library sync. The sync.Mutex type has only two public pointer methods, Lock and Unlock. Lock locks the current shared resource, and Unlock unlocks it.
When using a mutex, you must pay attention: after the resource operation is completed, you must unlock it, otherwise there will be abnormal process execution, deadlock and other problems. Usually with defer. After locking, use the defer statement immediately to ensure that the mutex is unlocked in time.
Lock() method
The Lock() method locks m. If the lock is already in use, the calling goroutine will block until the mutex becomes available.
Unlock() method
Unlock() method, unlock m. It is a runtime error if m is not locked on the entry being unlocked.
A locked mutex is not associated with a specific goroutine. Allow one goroutine to lock the mutex, and then schedule another goroutine to unlock the mutex.
sample code
Use goroutine to simulate the case of selling train tickets at 4 ticket gates. Four ticket outlets sell tickets at the same time, and critical resource data security issues will occur. Let's use a mutex to solve it. (Go language advocates the use of Channel to achieve data sharing, but it still provides traditional synchronization processing methods)
package main
import (
"fmt"
"math/rand"
"sync"
"time"
)
// 全局变量，表示票
var ticket = 10 //10张票
var mutex sync.Mutex //创建锁头
var wg sync.WaitGroup //同步等待组对象
func main() {
wg.Add(4)
go saleTickets("售票口1")
go saleTickets("售票口2")
go saleTickets("售票口3")
go saleTickets("售票口4")
wg.Wait() //main要等待
fmt.Println("程序结束了。。。")
}
func saleTickets(name string) {
rand.Seed(time.Now().UnixNano())
defer wg.Done()
for {
//上锁
mutex.Lock() //g2
if ticket > 0 {
//ticket 1 g1
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
fmt.Println(name, "售出：", ticket) // 1
ticket-- // 0
} else {
mutex.Unlock() //条件不满足，也要解锁
fmt.Println(name, "售罄，没有票了。。")
break
}
mutex.Unlock() //解锁
}
}
operation result:
售票口1 售出： 10
售票口1 售出： 9
售票口2 售出： 8
售票口4 售出： 7
售票口3 售出： 6
售票口1 售出： 5
售票口2 售出： 4
售票口4 售出： 3
售票口3 售出： 2
售票口1 售出： 1
售票口3 售罄，没有票了。。
售票口2 售罄，没有票了。。
售票口4 售罄，没有票了。。
售票口1 售罄，没有票了。。
程序结束了。。。
read-write lock
RWMutex (read-write lock)
Through the study of mutexes, we already know the concept and use of locks. It is mainly used to deal with critical resource issues in concurrency.
The sync package in the Go language package provides two lock types: sync.Mutex and sync.RWMutex. Among them, RWMutex is implemented based on Mutex, and the implementation of read-only lock uses a function similar to a reference counter.
RWMutex is a read/write mutex. A lock can be held by any number of readers or a single writer. A zero value for RWMutex is an unlocked mutex.
If one goroutine holds a rRWMutex for reading, and another goroutine might call lock, then no goroutine should expect to be able to acquire the read lock until the initial read lock is released. In particular, this prohibits recursive read locks. This is to ensure that the lock is eventually available; a blocked lock call will exclude new readers from acquiring the lock.
How do we understand read-write locks? When a goroutine acquires a write lock, other read locks or write locks will block until the write is unlocked; when a goroutine acquires a read lock, other read locks can still continue; when there are one or more read locks, the write lock Will wait for all read locks to unlock before being able to take write locks. So the purpose of the read lock (RLock) here is actually to tell the write lock: many people are reading the data, you stand aside for me, and you can write (write lock) after they read (read unlock). We can summarize it in the following three lines:
- Only one goroutine can acquire the write lock at a time.
- Any number of gorouintes can acquire read locks at the same time.
- Only write locks or read locks can exist at the same time (read and write are mutually exclusive).
Therefore, RWMutex is a read-write lock, which can add multiple read locks or a write lock, which is often used in scenarios where the number of reads is far greater than the number of writes .
The write lock of the read-write lock can only be locked once, and cannot be locked multiple times before unlocking. The read lock can be multiple times, but the number of read unlocks can only be one more than the number of read locks. In general, we do not recommend that the number of read unlocks exceed the number of read locks. frequency.
Basically follow two principles:
1. You can read at will, and multiple goroutines can read at the same time.
2. When writing, you can't do anything. Cannot read or write.
A read-write lock is a mutual exclusion lock for read and write operations. The biggest difference between it and ordinary mutex is that it can lock and unlock for read and write operations respectively. Read-write locks follow different access control rules than mutexes. Within the jurisdiction of the read-write lock, it allows any number of read operations to be performed simultaneously. But at a time, it only allows one write operation in progress.
And in the process of a certain write operation being carried out, the carrying out of the reading operation is also not allowed. That is to say, multiple write operations under the control of the read-write lock are mutually exclusive, and both write operations and read operations are also mutually exclusive. However, there is no mutually exclusive relationship between multiple read operations.
common method
RLock() method
func (rw *RWMutex) RLock()
Read locks, when there are write locks, read locks cannot be loaded. When there are only read locks or no locks, read locks can be loaded, and multiple read locks can be loaded, so it is suitable for the scenario of "read more and write less".
RUnlock() method
func (rw *RWMutex) RUnlock()
Read lock unlock, RUnlock undoes a single RLock call, it has no effect on other concurrent readers. If rw is not locked for reading, calling RUnlock will raise a runtime error.
Lock() method
func (rw *RWMutex) Lock()
Write lock, if there are other read locks and write locks before adding the write lock, the Lock will block until the lock is available. To ensure that the lock is finally available, the blocked Lock call will exclude the new lock from the obtained lock. Read locks, that is, write locks have higher permissions than read locks, and write locks are prioritized when there are write locks.
Unlock() method
func (rw *RWMutex) Unlock()
The write lock is unlocked, and a runtime error is raised if the write lock is not in place.
sample code
package main
import (
"fmt"
"sync"
"time"
)
var rwMutex *sync.RWMutex
var wg *sync.WaitGroup
func main() {
rwMutex = new(sync.RWMutex)
wg = new(sync.WaitGroup)
//wg.Add(2)
//
多个同时读取
//go readData(1)
//go readData(2)
wg.Add(3)
go writeData(1)
go readData(2)
go writeData(3)
wg.Wait()
fmt.Println("main..over...")
}
func writeData(i int) {
defer wg.Done()
fmt.Println(i, "开始写：write start。。")
rwMutex.Lock() //写操作上锁
fmt.Println(i, "正在写：writing。。。。")
time.Sleep(3 * time.Second)
rwMutex.Unlock()
fmt.Println(i, "写结束：write over。。")
}
func readData(i int) {
defer wg.Done()
fmt.Println(i, "开始读：read start。。")
rwMutex.RLock() //读操作上锁
fmt.Println(i, "正在读取数据：reading。。。")
time.Sleep(3 * time.Second)
rwMutex.RUnlock() //读操作解锁
fmt.Println(i, "读结束：read over。。。")
}
operation result:
3 开始写：write start。。
2 开始读：read start。。
3 正在写：writing。。。。
1 开始写：write start。。
3 写结束：write over。。
2 正在读取数据：reading。。。
2 读结束：read over。。。
1 正在写：writing。。。。
1 写结束：write over。。
main..over...
Final summary:
- Read locks cannot block read locks
- Read locks need to block write locks until all read locks are released
- Write locks need to block read locks until all write locks are released
- Write locks require blocking write locks
channel channel
Channels can be thought of as conduits for Goroutines to communicate. Similar to the flow of water in a pipe from one end to the other, data can be sent from one end to the other and received through a channel.
When talking about the concurrency of the Go language, we said that when multiple Goroutines want to share data, although the traditional synchronization mechanism is also provided, the Go language strongly recommends using the Channel channel to realize the communication between Goroutines. communication.
"Don't communicate by sharing memory, but share memory by communicating" This is a classic saying that is popular in the golang community
In Go language, to pass some data to another goroutine (coroutine), you can encapsulate this data into an object, and then pass the pointer of this object into a certain channel, and another goroutine reads this data from this channel. pointer, and handles the memory object it points to. From the language level, Go guarantees that only one goroutine can access the data in the channel at the same time, providing developers with an elegant and simple tool. Therefore, Go uses channels to communicate, and transfers memory data through communication, making memory data Pass between different goroutines instead of using shared memory to communicate.
what is a channel
The concept of channel
What is a channel, a channel is a channel between goroutines. It allows goroutines to communicate with each other.
Each channel has a type associated with it. The type is the data type that the channel allows to transfer. (The zero value of a channel is nil. A nil channel serves no purpose, so channels must be defined using methods similar to maps and slices.)
channel declaration
Declaring a channel has the same syntax as defining a variable:
//声明通道
var 通道名 chan 数据类型
//创建通道：如果通道为nil(就是不存在)，就需要先创建通道
通道名 = make(chan 数据类型)
Sample code:
package main
import "fmt"
func main() {
var a chan int
if a == nil {
fmt.Println("channel 是 nil 的, 不能使用，需要先创建通道。。")
a = make(chan int)
fmt.Printf("数据类型是： %T", a)
}
}
operation result:
channel 是 nil 的, 不能使用，需要先创建通道。。
数据类型是： chan int
A short statement is also possible:
a := make(chan int)
The data type of the channel
Channel is a reference type of data, when passed as a parameter, the memory address is passed.
Sample code:
package main
import (
"fmt"
)
func main() {
ch1 := make(chan int)
fmt.Printf("%T,%p\n",ch1,ch1)
test1(ch1)
}
func test1(ch chan int){
fmt.Printf("%T,%p\n",ch,ch)
}
operation result:
chan int,0xc00001e180
chan int,0xc00001e180
We can see that the addresses of ch and ch1 are the same, indicating that they are the same channel.
Channel Notes
When using the Channel channel, there are the following points to pay attention to:
-
1. Used for goroutine to deliver messages.
-
2. Channels, each has an associated data type,
nil chan, cannot be used, similar to nil map, cannot directly store key-value pairs -
3. Use the channel to transfer data: <-
chan <- data, send data to the channel. Write data to the channel
data <- chan, get data from the channel. read data from channel -
4. Blocking:
send data: chan <- data, blocked until another goroutine, read data to unblock
read data: data <- chan, also blocked. Until another goroutine writes data to unblock. -
5. The channel itself is synchronous, which means that only one goroutine can operate at the same time.
Finally: The channel is the connection between goroutines, so the sending and receiving of the channel must be in different goroutines.
channel usage syntax
send and receive
Syntax for sending and receiving:
data := <- a // read from channel a
a <- data // write to channel a
The direction of the arrow on the channel specifies whether data is sent or received.
in addition:
v, ok := <- a //从一个channel中读取
Send and receive are blocking by default
A channel to send and receive data is blocked by default. When a data is sent to a channel, block in the send statement until another Goroutine reads data from the channel. Conversely, when reading data from a channel, the read is blocked until a Goroutine writes data to the channel.
The nature of these channels is to help Goroutines communicate efficiently without using explicit locks or condition variables that are very common in other programming languages.
Sample code:
package main
import "fmt"
func main() {
var ch1 chan bool //声明，没有创建
fmt.Println(ch1) //<nil>
fmt.Printf("%T\n", ch1) //chan bool
ch1 = make(chan bool) //0xc0000a4000,是引用类型的数据
fmt.Println(ch1)
go func() {
for i := 0; i < 10; i++ {
fmt.Println("子goroutine中，i：", i)
}
// 循环结束后，向通道中写数据，表示要结束了。。
ch1 <- true
fmt.Println("结束。。")
}()
data := <-ch1 // 从ch1通道中读取数据
fmt.Println("data-->", data)
fmt.Println("main。。over。。。。")
}
operation result:
<nil>
chan bool
0xc000086120
子goroutine中，i： 0
子goroutine中，i： 1
子goroutine中，i： 2
子goroutine中，i： 3
子goroutine中，i： 4
子goroutine中，i： 5
子goroutine中，i： 6
子goroutine中，i： 7
子goroutine中，i： 8
子goroutine中，i： 9
结束。。
data--> true
main。。over。。。。
In the above program, we first created a chan bool channel. Then start a child Goroutine, and print 10 numbers in a loop. Then we write the input true to the channel ch1. Then in the main goroutine, we read data from ch1. This line of code is blocking, which means that the main goroutine will not execute to the next line of code until the child goroutine writes data to the channel. Therefore, we can realize the communication between the child goroutine and the main goroutine through the channel. When the child goroutine finishes executing, the main goroutine will block because it reads the data in ch1. This ensures that the child goroutine will be executed first. This eliminates the need for time. In the previous program, we either put the main goroutine to sleep to prevent the main goroutine from exiting. Either use WaitGroup to ensure that the child goroutine is executed first before the main goroutine ends.
Sample code: The following code adds sleep to better understand channel blocking
package main
import (
"fmt"
"time"
)
func main() {
ch1 := make(chan int)
done := make(chan bool) // 通道
go func() {
fmt.Println("子goroutine执行。。。")
time.Sleep(3 * time.Second)
data := <-ch1 // 从通道中读取数据
fmt.Println("data：", data)
done <- true
}()
// 向通道中写数据。。
time.Sleep(5 * time.Second)
ch1 <- 100
<-done
fmt.Println("main。。over")
}
operation result:
子goroutine执行。。。
data： 100
main。。over
As another example, the following program will print the sum of the squares and the sum of the cubes of each digit of a number.
package main
import (
"fmt"
)
func calcSquares(number int, squareop chan int) {
sum := 0
for number != 0 {
digit := number % 10
sum += digit * digit
number /= 10
}
squareop <- sum
}
func calcCubes(number int, cubeop chan int) {
sum := 0
for number != 0 {
digit := number % 10
sum += digit * digit * digit
number /= 10
}
cubeop <- sum
}
func main() {
number := 123
sqrch := make(chan int)
cubech := make(chan int)
go calcSquares(number, sqrch)
go calcCubes(number, cubech)
squares, cubes := <-sqrch, <-cubech
fmt.Println("Final output", squares, cubes)
}
operation result:
Final output 14 36
deadlock
An important factor to consider when using channels is deadlock. If a goroutine sends data on a channel, it is expected that other goroutines should receive the data. If this does not happen, the program will deadlock at runtime.
Similarly, if a Goroutine is waiting to receive data from a channel, then some other Goroutine will write data on that channel, otherwise the program will deadlock.
close channel
The sender can close the channel to notify the receiver that no more data will be sent to the channel.
close(ch)
Receivers can use additional variables when receiving data from a channel to check whether the channel has been closed.
Grammatical structures:
v, ok := <- ch
Similar to map operation, store key, value key-value pairs
v,ok := map[key] // Get the value from the map according to the key, if the key exists, v is the corresponding data, if the key does not exist, v is the default value
In the above statement, if the value of ok is true, it means that a data value was successfully read from the channel. If ok is false, it means we are reading from a closed channel. Values read from a closed channel will be the zero value of the channel type.
For example, if the channel is an int channel, then the value received from the closed channel will be 0.
Sample code:
package main
import (
"fmt"
"time"
)
func main() {
ch1 := make(chan int)
go sendData(ch1)
/*
子goroutine，写出数据10个
每写一个，阻塞一次，主程序读取一次，解除阻塞
主goroutine：循环读
每次读取一个，堵塞一次，子程序，写出一个，解除阻塞
发送发，关闭通道的--->接收方，接收到的数据是该类型的零值，以及false
*/
//主程序中获取通道的数据
for {
time.Sleep(1 * time.Second)
v, ok := <-ch1 //其他goroutine，显示的调用close方法关闭通道。
if !ok {
fmt.Println("已经读取了所有的数据，", ok, v)
break
}
fmt.Println("取出数据：", v, ok)
}
fmt.Println("main...over....")
}
func sendData(ch1 chan int) {
// 发送方：10条数据
for i := 0; i < 10; i++ {
ch1 <- i //将i写入通道中
}
close(ch1) //将ch1通道关闭了。
}
operation result
取出数据： 0 true
取出数据： 1 true
取出数据： 2 true
取出数据： 3 true
取出数据： 4 true
取出数据： 5 true
取出数据： 6 true
取出数据： 7 true
取出数据： 8 true
取出数据： 9 true
已经读取了所有的数据， false 0
main...over....
In the above program, the send Goroutine writes 0 to 9 to the chl channel and then closes the channel. There is an infinite loop in the main function. It checks if the channel is closed using the variable ok after sending data. If ok is false, it means the channel is closed, so the loop ends. You can also print the received value and the ok value.
range loop on channel
We can get data from the channel in a loop until the channel is closed. The for range form of a for loop can be used to receive values from a channel until it is closed.
Use range loop, sample code:
package main
import (
"fmt"
"time"
)
func main() {
ch1 := make(chan int)
go sendData(ch1)
// for循环的for range形式可用于从通道接收值，直到它关闭为止。
for v := range ch1 {
fmt.Println("读取数据：", v)
}
fmt.Println("main..over.....")
}
func sendData(ch1 chan int) {
for i := 0; i < 10; i++ {
time.Sleep(1 * time.Second)
ch1 <- i
}
close(ch1) //通知对方，通道关闭
}
operation result:
读取数据： 0
读取数据： 1
读取数据： 2
读取数据： 3
读取数据： 4
读取数据： 5
读取数据： 6
读取数据： 7
读取数据： 8
读取数据： 9
main..over.....
buffer channel
unbuffered channel
All previously learned channels are essentially unbuffered. Sending and receiving on an unbuffered channel is blocking.
A sending operation corresponds to a receiving operation. For a goroutine, its sending is blocked until another goroutine receives it. Likewise, for receiving, it blocks until another goroutine sends.
buffer channel
A buffered channel is just a channel with a buffer. Sending to a buffered channel only blocks when the buffer is full. Similarly, messages received from buffered channels will only block when the buffer is empty.
Buffered channels can be created by passing an additional capacity argument to the make function, which specifies the size of the buffer.
grammar:
ch := make(chan type, capacity)
The capacity of the above syntax should be greater than 0 in order for the channel to have a buffer. By default, an unbuffered channel has a capacity of 0, so the capacity parameter was omitted when the channel was previously created.
sample code
In the following code, the chan channel has a buffer.
package main
import (
"fmt"
"strconv"
"time"
)
func main() {
/*
非缓存通道：make(chan T)
缓存通道：make(chan T ,size)
缓存通道，理解为是队列：
非缓存，发送还是接受，都是阻塞的
缓存通道,缓存区的数据满了，才会阻塞状态。。
*/
ch := make(chan string, 4)
go sendData3(ch)
for {
time.Sleep(time.Second / 2)
v, ok := <-ch
if !ok {
fmt.Println("读完了，，", ok)
break
}
fmt.Println("\t读取的数据是：", v)
}
fmt.Println("main...over...")
}
func sendData3(ch chan string) {
for i := 0; i < 10; i++ {
ch <- "数据" + strconv.Itoa(i)
fmt.Println("子goroutine，写出第", i, "个数据")
}
close(ch)
}
operation result:
子goroutine，写出第 0 个数据
子goroutine，写出第 1 个数据
子goroutine，写出第 2 个数据
子goroutine，写出第 3 个数据
读取的数据是： 数据0
子goroutine，写出第 4 个数据
子goroutine，写出第 5 个数据
读取的数据是： 数据1
子goroutine，写出第 6 个数据
读取的数据是： 数据2
读取的数据是： 数据3
子goroutine，写出第 7 个数据
读取的数据是： 数据4
子goroutine，写出第 8 个数据
读取的数据是： 数据5
子goroutine，写出第 9 个数据
读取的数据是： 数据6
读取的数据是： 数据7
读取的数据是： 数据8
读取的数据是： 数据9
读完了，， false
main...over...
directional channel
two-way channel
Channels, channels, are used to implement communication between goroutines. One goroutine can send data to the channel, and another goroutine can get data from the channel. The channels we have studied so far can both send data and read data. We also call this channel a two-way channel.
data := <- a // read from channel a
a <- data // write to channel a
package main
import "fmt"
func main() {
ch1 := make(chan string) // 双向，可读，可写
done := make(chan bool)
go sendData(ch1, done)
data := <-ch1 //阻塞
fmt.Println("子goroutine传来：", data)
ch1 <- "我是main。。" // 阻塞
<-done
fmt.Println("main...over....")
}
// 子goroutine-->写数据到ch1通道中
// main goroutine-->从ch1通道中取
func sendData(ch1 chan string, done chan bool) {
ch1 <- "我是小明" // 阻塞
data := <-ch1 // 阻塞
fmt.Println("main goroutine传来：", data)
done <- true
}
operation result:
子goroutine传来： 我是小明
main goroutine传来： 我是main。。
main...over....
one way channel
One-way channel, that is, directional channel.
The channels we learned before are all two-way channels, and we can receive or send data through these channels. We can also create unidirectional channels, which can only send or receive data.
Create a channel that can only send data, sample code:
Sample code:
package main
import "fmt"
func main() {
/*
单向：定向
chan <- T,
只支持写，
<- chan T,
只读
*/
ch1 := make(chan int)//双向，读，写
//ch2 := make(chan <- int) // 单向，只写，不能读
//ch3 := make(<- chan int) //单向，只读，不能写
//ch1 <- 100
//data :=<-ch1
//ch2 <- 1000
//data := <- ch2
//fmt.Println(data)
// <-ch2 //invalid operation: <-ch2 (receive from send-only type chan<- int)
//ch3 <- 100
// <-ch3
// ch3 <- 100 //invalid operation: ch3 <- 100 (send to receive-only type <-chan int)
//go fun1(ch2)
go fun1(ch1)
data:= <- ch1
fmt.Println("fun1中写出的数据是：",data)
//fun2(ch3)
go fun2(ch1)
ch1 <- 200
fmt.Println("main。。over。。")
}
//该函数接收，只写的通道
func fun1(ch chan <- int){
// 函数内部，对于ch只能写数据，不能读数据
ch <- 100
fmt.Println("fun1函数结束。。")
}
func fun2(ch <-chan int){
//函数内部，对于ch只能读数据，不能写数据
data := <- ch
fmt.Println("fun2函数，从ch中读取的数据是：",data)
}
operation result:
fun1函数结束。。
fun1中写出的数据是： 100
fun2函数，从ch中读取的数据是： 200
main。。over。。
Channel-related functions in the time package
The main thing is the timer. The Timer in the standard library allows users to define their own timeout logic, which is especially convenient when dealing with the timeout of selecting multiple channels or the timeout of reading and writing of a single channel.
Timer is a one-time time-triggered event, which is different from Ticker, which continuously triggers time events at a certain interval.
Common creation methods of Timer:
t:= time.NewTimer(d)
t:= time.AfterFunc(d, f)
c:= time.After(d)
Although the creation methods are different, the principle is the same.
Timer has 3 elements:
Timing time: that is the d
trigger action: that is the f
time channel: that is tC
time.NewTimer()
NewTimer() creates a new timer that will send the current time after at least d duration on its channel. Its return value is a Timer.
source code:
// NewTimer creates a new Timer that will send
// the current time on its channel after at least duration d.
func NewTimer(d Duration) *Timer {
c := make(chan Time, 1)
t := &Timer{
C: c,
r: runtimeTimer{
when: when(d),
f: sendTime,
arg: c,
},
}
startTimer(&t.r)
return t
}
We can see from the source code that first a channel is created, the associated type is Time, and then a Timer is created and returned.
- Used to call a function or calculate an expression after a specified Duration type of time.
- If you just want to execute after a specified time, use time.Sleep()
- Using NewTimer(), you can return the Timer type to cancel the timer before the timer expires
- The timer will not expire until a value is sent using <- timer.C
Sample code:
package main
import (
"fmt"
"time"
)
func main() {
/*
func NewTimer(d Duration) *Timer
创建一个计时器：d时间以后触发，go触发计时器的方法比较特别，就是在计时器的channel中发送值
*/
//新建一个计时器：timer
timer := time.NewTimer(3 * time.Second)
fmt.Printf("%T\n", timer) //*time.Timer
fmt.Println(time.Now()) //2023-07-07 16:26:45.5207225 +0800 CST m=+0.001542901
//此处在等待channel中的信号，执行此段代码时会阻塞3秒
ch2 := timer.C //<-chan time.Time
fmt.Println(<-ch2) //2023-07-07 16:26:48.5308961 +0800 CST m=+3.011716501
}
timer.Stop
Timer stopped:
Sample code:
package main
import (
"fmt"
"time"
)
func main() {
//新建计时器，一秒后触发
timer2 := time.NewTimer(3 * time.Second)
//新开启一个线程来处理触发后的事件
go func() {
//等触发时的信号
<-timer2.C
fmt.Println("Timer 2 结束。。")
}()
//由于上面的等待信号是在新线程中，所以代码会继续往下执行，停掉计时器
time.Sleep(1 * time.Second)
stop := timer2.Stop()
if stop {
fmt.Println("Timer 2 停止。。")
}
}
operation result:
Timer 2 停止。。
time.After()
After waiting for the duration, the current time is then sent on the returned channel. It is equivalent to NewTimer(d).C. The garbage collector will not restore the underlying timer until the timer fires. If efficiency is a problem, use NewTimer instead, and call Timer. If the timer is no longer needed, stop it.
Sample code:
package main
import (
"fmt"
"time"
)
func main() {
/*
func After(d Duration) <-chan Time
返回一个通道：chan，存储的是d时间间隔后的当前时间。
*/
ch1 := time.After(3 * time.Second) //3s后
fmt.Printf("%T\n", ch1) // <-chan time.Time
fmt.Println(time.Now()) //2023-07-07 16:36:10.8553299 +0800 CST m=+0.001792901
time2 := <-ch1
fmt.Println(time2) //2023-07-07 16:36:13.8602885 +0800 CST m=+3.006751501
}
select statement
select is a control structure in Go. The select statement is similar to the switch statement, but select randomly executes a runnable case. If there are no cases to run, it will block until there are cases to run.
Grammatical structures
The grammatical structure of the select statement is very similar to the switch statement, and there are also case statements and default statements:
select {
case communication clause :
statement(s);
case communication clause :
statement(s);
/* 你可以定义任意数量的 case */
default : /* 可选 */
statement(s);
}
illustrate:
-
Each case must be a communication
-
All channel expressions are evaluated
-
All expressions sent will be evaluated
-
If there are multiple cases that can be run, select will randomly and fairly select one for execution. Others will not execute.
-
otherwise:
If there is a default clause, the statement is executed.
Without the default clause, select will block until some communication can run; Go will not re-evaluate the channel or value.
sample code
package main
import (
"fmt"
"time"
)
func main() {
ch1 := make(chan int)
ch2 := make(chan int)
go func() {
time.Sleep(2 * time.Second)
ch2 <- 200
}()
go func() {
time.Sleep(2 * time.Second)
ch1 <- 100
}()
select {
case num1 := <-ch1:
fmt.Println("ch1中取数据。。", num1)
case num2, ok := <-ch2:
if ok {
fmt.Println("ch2中取数据。。", num2)
} else {
fmt.Println("ch2通道已经关闭。。")
}
}
}
Running result: the first case may be executed and 100 printed, or the second case may be executed and 200 printed. (Run it several times, the result will be different)
The select statement combines the time package and chan related functions, sample code:
package main
import (
"fmt"
"time"
)
func main() {
ch1 := make(chan int)
ch2 := make(chan int)
go func() {
ch1 <- 100
}()
select {
case <-ch1:
fmt.Println("case1可以执行。。")
case <-ch2:
fmt.Println("case2可以执行。。")
case <-time.After(3 * time.Second):
fmt.Println("case3执行。。timeout。。")
default:
fmt.Println("执行了default。。")
}
}
Running result: case1 can be executed. . Or execute default. .
CSP model of Go language
The two biggest highlights of the go language are goroutine and chan. The typical application of the combination of the two, CSP, is basically a parallel development artifact recognized by everyone, which simplifies the development difficulty of parallel programs. Let's take a look at CSP.
What is CSP
CSP is the abbreviation of Communicating Sequential Process. Chinese can be called a communication sequential process. It is a concurrent programming model and a very powerful concurrent data model. It was proposed in the 1970s to describe two independent concurrent entities. A concurrency model for communicating through shared communication channels. Compared with the Actor model, the channel in CSP is the first type of object. It does not pay attention to the entity that sends the message, but to the channel used when sending the message.
Strictly speaking, CSP is a formal language (similar to ℷ calculus), which is used to describe the interaction mode in concurrent systems. Therefore, it has become the theoretical source of many concurrency-oriented programming languages, and derived Occam/Limbo/Golang …
As for programming languages, such as Golang, only a small part of CSP is actually used, that is, Process/Channel in theory (corresponding to goroutine/channel in language): there is no affiliation between these two concurrent primitives, and Process You can subscribe to any Channel, and the Channel doesn't care which Process is using it for communication; the Process reads and writes around the Channel, forming a set of ordered blocking and predictable concurrency models.
Golang CSP
Unlike mainstream languages that use shared memory for concurrency control, Go uses the CSP model. This is a concurrency model used to describe two independent concurrent entities communicating through a shared communication Channel (pipeline).
Golang borrows some concepts of the CSP model to achieve concurrency for theoretical support. In fact, the go language does not fully realize all the theories of the CSP model, but only borrows the two concepts of process and channel. The performance of process in go language is that goroutine is an entity that is actually executed concurrently, and each entity realizes data sharing through channel communication.
The CSP model of the Go language is implemented by the coroutine Goroutine and the channel Channel:
- Go coroutine goroutine: It is a light-weight thread. It is not an operating system thread, but uses an operating system thread in segments, and realizes cooperative scheduling through the scheduler. It is a green thread, a micro-thread, which is also different from a Coroutine coroutine, and can start a new micro-thread after a blockage is found.
- Channel channel: Unix-like Pipe, used for communication and synchronization between coroutines. Although the coroutines are decoupled, they are coupled with the Channel.
Channel
Goroutine and channel are the two cornerstones of concurrent programming in Go language. Goroutine is used to execute concurrent tasks, and channel is used for synchronization and communication between goroutines.
Channel sets up a pipeline between gouroutines, transmits data in the pipeline, and realizes communication between gouroutines; because it is thread-safe, it is very convenient to use; channel also provides the "first in first out" feature; it can also affect goroutines blocking and wakeup.
I believe everyone must have seen a sentence:
Do not communicate by sharing memory; instead, share memory by communicating.
Don't communicate by sharing memory, share memory by communicating.
This is the concurrency philosophy of Go, which relies on the CSP model and is implemented based on channels.
Channel implements CSP
Channel is a very important type in Go language and the first object in Go. Through channels, Go implements memory sharing through communication. Channel is an important means of transferring data and synchronizing between multiple goroutines.
Using atomic functions and read-write locks can ensure the security of shared access to resources, but using channels is more elegant.
channel literally means "channel", similar to a pipe in Linux. The syntax for declaring a channel is as follows:
chan T // 声明一个双向通道
chan<- T // 声明一个只能用于发送的通道
<-chan T // 声明一个只能用于接收的通道
The declaration of a one-way channel, denoted <-by , indicates the direction of the channel. As long as you understand that the writing order of the code is from left to right, you can immediately grasp the direction of the channel.
Because channel is a reference type, its value is nil before it is initialized, and channel is initialized using the make function. An int value can be passed to it, representing the size (capacity) of the channel buffer, and a buffered channel is constructed; if no pass or 0 is passed, a non-buffered channel is constructed.
There are some differences between the two: a non-buffered channel cannot buffer elements, and the order of operations on it is "send->receive->send->receive->...", if you want to send 2 elements to a non-buffered channel continuously , and if there is no reception, it will definitely be blocked for the first time; for buffered channel operations, it should be "relaxed", because after all, it has a "buffering" halo.
The sending and receiving operations on chan will be converted into the underlying sending and receiving functions during compilation.
Channel is divided into two types: buffered and unbuffered. The operation on the unbuffered channel can actually be regarded as "synchronous mode", and the buffered channel is called "asynchronous mode".
In synchronous mode, the sender and receiver must be ready for synchronization, and only when both are ready, data can be transmitted between them (as you will see later, it is actually a memory copy). Otherwise, any party that performs sending or receiving operations first will be suspended and wait for the appearance of the other party to be woken up.
In asynchronous mode, both sending and receiving operations can proceed smoothly as long as the buffer slot is available (with remaining capacity). Otherwise, one side of the operation (such as writing) will also be suspended until the opposite operation (such as receiving) will not be woken up.
To summarize: In synchronous mode, the operation must be paired between the sender and the receiver before the operation will succeed, otherwise it will be blocked; in asynchronous mode, the operation will succeed only if there is remaining capacity in the buffer slot, otherwise it will also be blocked.
To put it simply, the CSP model is composed of concurrently executing entities (threads or processes or coroutines). Entities communicate by sending messages. Here, channels are
used when sending messages, or channels.
The key to the CSP model is to focus on the channel, not the entity sending the message. The Go language implements part of the theory of CSP, goroutine corresponds to the concurrent execution entity in CSP, and channel corresponds to the channel in CSP.
Goroutine
Goroutine is the entity that actually executes concurrently. Its bottom layer uses coroutine to achieve concurrency. Coroutine is a user thread running in user mode, similar to greenthread. The reason why go bottom layer chooses to use coroutine is because it has the following characteristics :
- User space avoids the cost caused by switching between kernel mode and user mode
- Can be scheduled by language and framework layers
- Smaller stack space allows creating a large number of instances
It can be seen that the scheduling of the second user space thread is not done by the operating system, like the greenthread used in java 1.3 is uniformly scheduled by the JVM (later java has been changed to a kernel thread), and in ruby Fiber (semi-coroutine) needs to be scheduled in the re-run, while goroutine provides a scheduler at the golang level, and encapsulates the network IO library, shields complex details, and provides unified grammatical keyword support to the outside world , which simplifies the cost of concurrent program writing.
Goroutine scheduler
Go Concurrency Scheduling: GPM Model
Go builds a unique two-level threading model on top of the kernel threads provided by the operating system. The goroutine mechanism implements the M:N thread model. The goroutine mechanism is an implementation of a coroutine. The built-in scheduler of golang allows each CPU in a multi-core CPU to execute a coroutine.
at last
Golang's channel isolates goroutines, and you can focus on the channel during concurrent programming. To a certain extent, this is quite similar to the decoupling function of the message queue. If you are interested, let's take a look at the source code of the channel, which is quite useful for a deeper understanding of the channel.
Go implements the CSP communication model through channels, which are mainly used for message passing and event notification between goroutines.
With channels and goroutines, Go's concurrent programming becomes extremely easy and safe, allowing programmers to focus on business and improve development efficiency.
You know, technology is not the most important thing, it is just a tool to realize business. An efficient development language allows you to save the time you save to do more meaningful things, such as writing articles.
reflection
introduce
First look at the definition of reflection given by Rob Pike in the official Doc:
Reflection in computing is the ability of a program to examine its own structure, particularly through types; it’s a form of metaprogramming. It’s also a great source of confusion.
(在计算机领域，反射是一种让程序——主要是通过类型——理解其自身结构的一种能力。它是元编程的组成之一，同时它也是一大引人困惑的难题。)
Definition from Wikipedia:
In computer science, reflection refers to the ability of a computer program to access, detect and modify its own state or behavior at runtime (Run time). Metaphorically speaking, reflection is the ability of a program to "observe" and modify its behavior while it is running.
The reflection models of different languages are not the same, and some languages do not support reflection yet. Reflection is defined in the "Go Language Bible" as follows:
The Go language provides a mechanism to update variables, check their values, and call their methods at runtime, but the specific types of these variables are not known at compile time, which is called the reflection mechanism.
Why use reflection
2 common scenarios that require reflection:
- Sometimes you need to write a function, but you don’t know the type of the parameter passed to you, it may be that there is no agreement; it may also be that there are many types passed in, and these types cannot be represented uniformly. This is where reflection comes in handy.
- Sometimes it is necessary to decide which function to call based on certain conditions, such as user input. At this time, it is necessary to reflect on the function and the parameters of the function, and execute the function dynamically during operation.
But for reflection, there are still a few reasons why it is not recommended to use reflection:
- Code related to reflection is often difficult to read. In software engineering, code readability is also a very important indicator.
- The Go language is a static language. During the coding process, the compiler can detect some type errors in advance, but it can't do anything about the reflection code. Therefore, the code containing reflection is likely to run for a long time before making an error. At this time, it often panics directly, which may cause serious consequences.
- Reflection has a relatively large impact on performance, running one to two orders of magnitude slower than normal code. Therefore, for code in a project that is at a critical position in operating efficiency, try to avoid using reflection features.
related basis
How is reflection implemented? We have learned about interface before, which is a very powerful tool for abstraction in Go language. When assigning an entity type to an interface variable, the interface will store the type information of the entity, and reflection is realized through the type information of the interface, and reflection is based on the type.
The Go language defines various types in the reflect package, and implements various functions of reflection, through which the information of the type can be detected and the value of the type can be changed at runtime. Before going into a more detailed understanding, we need to revisit some features related to the Go language, the so-called learning from the past, and understand how its reflection mechanism is used from these features.
| features | illustrate |
|---|---|
| The go language is a statically typed language. | The type has been determined at compile time. For example, for the redefined type of the basic data type, it is necessary to confirm what type is returned during reflection. |
| empty interface interface{} | The reflection mechanism of go is carried out through the interface, and the empty interface similar to Java's Object can interact with any type, so the reflection of basic data types and so on also directly uses this feature |
| Types of the Go language: |
-
Variables include (type, value) two parts
Understand this and you will know why nil != nil
-
type includes static type and concrete type. Simply put, static type is the type you see when encoding (such as int, string), and concrete type is the type seen by the runtime system
-
Whether the type assertion can succeed depends on the concrete type of the variable, not the static type. Therefore, a reader variable can also be type asserted as a writer if its concrete type also implements the write method.
The reflection of the Go language is based on the type. The type of the variable of the specified type in Golang is static (that is, the type of the specified variable such as int, string, etc., its type is static type), and it has been determined when the variable is created. , reflection is mainly related to the interface type of Golang (its type is concrete type), and only the interface type has reflection.
In the implementation of Golang, each interface variable has a corresponding pair, and the value and type of the actual variable are recorded in the pair:
(value, type)
value is the actual variable value, and type is the type of the actual variable. A variable of type interface{} contains 2 pointers, one pointer points to the type of value [corresponding to concrete type], and the other pointer points to the actual value [corresponding to value].
For example, create a variable of type *os.File and assign it to an interface variable r:
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
var r io.Reader
r = tty
The following information will be recorded in the pair of the interface variable r: (tty, *os.File), this pair is invariable during the continuous assignment of the interface variable, and the interface variable r is assigned to another interface variable w:
var w io.Writer
w = r.(io.Writer)
The pair of the interface variable w is the same as the pair of r, both are: (tty, *os.File), even if w is an empty interface type, the pair remains unchanged.
The existence of interface and its pair is the prerequisite for implementing reflection in Golang. Understanding pair makes it easier to understand reflection. Reflection is a mechanism for detecting pairs stored in interface variables (value value; type concrete type).
So we need to understand two basic concepts Type and Value, which are also the two most important types in the reflect space in the Go language package.
use of reflection
The package we generally use is the reflect package.
TypeOf和ValueOf
Since reflection is a mechanism for detecting pairs stored inside interface variables (value value; type concrete type). So what is the way in Golang's reflect reflection package that allows us to directly obtain the information inside the variable? It provides two types (or two methods) that allow us to easily access the content of interface variables, namely reflect.ValueOf() and reflect.TypeOf(), see the official explanation:
// ValueOf returns a new Value initialized to the concrete value
// stored in the interface i. ValueOf(nil) returns the zero
func ValueOf(i interface{
}) Value {
...}
翻译一下：ValueOf用来获取输入参数接口中的数据的值，如果接口为空则返回0
// TypeOf returns the reflection Type that represents the dynamic type of i.
// If i is a nil interface value, TypeOf returns nil.
func TypeOf(i interface{
}) Type {
...}
翻译一下：TypeOf用来动态获取输入参数接口中的值的类型，如果接口为空则返回nil
reflect.TypeOf() is to get the type in the pair, and reflect.ValueOf() is to get the value in the pair.
First, you need to convert it into a reflect object (reflect.Type or reflect.Value, and call different functions according to different situations.
t := reflect.TypeOf(i) //得到类型的元数据,通过t我们能获取类型定义里面的所有元素
v := reflect.ValueOf(i) //得到实际的值，通过v我们获取存储在里面的值，还可以去改变值
Sample code:
package main
import (
"fmt"
"reflect"
)
func main() {
//反射操作：通过反射，可以获取一个接口类型变量的 类型和数值
var x = 3.14
fmt.Println("type:", reflect.TypeOf(x)) //type: float64
fmt.Println("value:", reflect.ValueOf(x)) //value: 3.4
fmt.Println("-------------------")
//根据反射的值，来获取对应的类型和数值
v := reflect.ValueOf(x)
fmt.Println("kind is float64: ", v.Kind() == reflect.Float64)
fmt.Println("type : ", v.Type())
fmt.Println("value : ", v.Float())
}
operation result:
type: float64
value: 3.14
-------------------
kind is float64: true
type : float64
value : 3.14
illustrate
- reflect.TypeOf: It directly gives the type we want, such as float64, int, various pointers, struct and so on.
- reflect.ValueOf: It directly gives the specific value we want, such as the specific value of 1.2345, or the value of the structure struct like &{1 "Allen.Wu" 25}
- That is to say, reflection can convert "interface type variable" into "reflection type object". The reflection type refers to reflect.Type and reflect.Value.
Both Type and Value contain a large number of methods, and the first useful method should be Kind, which returns the specific information of the type: Uint, Float64, etc. The Value type also includes a series of type methods, such as Int(), for returning the corresponding value. The following are the types of Kind:
// A Kind represents the specific kind of type that a Type represents.
// The zero Kind is not a valid kind.
type Kind uint
const (
Invalid Kind = iota
Bool
Int
Int8
Int16
Int32
Int64
Uint
Uint8
Uint16
Uint32
Uint64
Uintptr
Float32
Float64
Complex64
Complex128
Array
Chan
Func
Interface
Map
Ptr
Slice
String
Struct
UnsafePointer
)
Get interface information from relfect.Value
After executing reflect.ValueOf(interface), you get a variable of type "relfect.Value". You can get the real content of the interface variable through its own Interface() method, and then convert it to the original There are real types. However, we may know the original type, or we may not know the original type. Therefore, the following two cases will be explained.
Known original type [perform "casting"]
After the known type is converted to its corresponding type, the method is as follows, directly through the Interface method and then forced conversion, as follows:
realValue := value.Interface().(已知的类型)
Sample code:
package main
import (
"fmt"
"reflect"
)
func main() {
var num float64 = 1.2345
pointer := reflect.ValueOf(&num)
value := reflect.ValueOf(num)
// 可以理解为“强制转换”，但是需要注意的时候，转换的时候，如果转换的类型不完全符合，则直接panic
// Golang 对类型要求非常严格，类型一定要完全符合
// 如下两个，一个是*float64，一个是float64，如果弄混，则会panic
convertPointer := pointer.Interface().(*float64)
convertValue := value.Interface().(float64)
fmt.Println(convertPointer)
fmt.Println(convertValue)
}
operation result:
0xc000098000
1.2345
illustrate
- When converting, if the converted type does not fully match, it will panic directly, and the type requirements are very strict!
- When converting, it is necessary to distinguish whether it is a pointer type or a non-pointer type
- That is to say, reflection can reconvert "reflection type object" into "interface type variable"
Unknown original type [traverse and detect its Filed]
In many cases, we may not know the specific type, so what should we do at this time? We need to traverse and detect its Filed to know, the example is as follows:
package main
import (
"fmt"
"reflect"
)
type Person struct {
Name string
Age int
Sex string
}
func (p Person) Say(msg string) {
fmt.Println("hello,", msg)
}
func (p Person) PrintInfo() {
fmt.Printf("姓名：%s，年龄：%d，性别：%s\n", p.Name, p.Age, p.Sex)
}
func main() {
p1 := Person{
"王二狗", 30, "男"}
GetMessage(p1)
}
// GetMessage 获取input的信息
func GetMessage(input interface{
}) {
getType := reflect.TypeOf(input) //先获取input的类型
fmt.Println("get Type is :", getType.Name()) //Person
fmt.Println("get Kind is :", getType.Kind()) //struct
getValue := reflect.ValueOf(input)
fmt.Println("get all Fields is :", getValue) //{王二狗 30 男}
//获取字段
/*
step1：先获取Type对象：reflect.Type，
NumField()
Field(index)
step2：通过Filed()获取每一个Filed字段
step3：Interface()，得到对应的Value
*/
for i := 0; i < getType.NumField(); i++ {
filed := getType.Field(i)
value := getValue.Field(i).Interface() //获取第一个数值
fmt.Printf("字段名称：%s，字段类型：%s，字段数值：%v\n", filed.Name, filed.Type, value)
}
//获取方法
for i := 0; i < getType.NumMethod(); i++ {
method := getType.Method(i)
fmt.Printf("方法名称：%s，方法类型：%v\n", method.Name, method.Type)
}
}
operation result:
get Type is : Person
get Kind is : struct
get all Fields is : {王二狗 30 男}
字段名称：Name，字段类型：string，字段数值：王二狗
字段名称：Age，字段类型：int，字段数值：30
字段名称：Sex，字段类型：string，字段数值：男
方法名称：PrintInfo，方法类型：func(main.Person)
方法名称：Say，方法类型：func(main.Person, string)
illustrate
From the running results, we can know that the steps to obtain the specific variable and its type of the interface of unknown type are:
- First get the reflect.Type of the interface, and then traverse through NumField
- Then get its Field through the Field of reflect.Type
- Finally, get the corresponding value through Field's Interface()
From the running results, we can know that the steps to obtain the method (function) of the interface of unknown type are:
- First get the reflect.Type of the interface, and then traverse through NumMethod
- Then obtain the corresponding real method (function) through the Method of reflect.Type
- Finally, take the Name and Type of the result to know the specific method name
- That is to say, reflection can reconvert "reflection type object" into "interface type variable"
- The nesting of struct or struct is the same judgment processing method
Set the value of the actual variable by reflect.ValueOf
reflect.Value is obtained through reflect.ValueOf(X). Only when X is a pointer, can the value of the actual variable X be modified through reflect.Value, that is, to modify an object of reflection type, its value must be "addressable".
A method is needed here:
func (v Value) Elem() Value
The explanation is: Elem returns the value contained in the interface v or the value pointed to by the pointer v. It panics if v is not of type interface or ptr. Returns a value of zero if v is zero.
If your variable is a pointer, map, slice, channel, Array. Then you can use reflect.Typeof(v).Elem() to determine the contained type.
package main
import (
"reflect"
"fmt"
)
func main() {
//1.“接口类型变量”=>“反射类型对象”
var circle float64 = 6.28
var icir interface{
}
icir = circle
fmt.Println("Reflect : circle.Value = ", reflect.ValueOf(icir)) //Reflect : circle.Value = 6.28
fmt.Println("Reflect : circle.Type = ", reflect.TypeOf(icir)) //Reflect : circle.Type = float64
// 2. “反射类型对象”=>“接口类型变量
v1 := reflect.ValueOf(icir)
fmt.Println(v1) //6.28
fmt.Println(v1.Interface()) //6.28
y := v1.Interface().(float64)
fmt.Println(y) //6.28
//v1.SetFloat(4.13) //panic: reflect: reflect.Value.SetFloat using unaddressable value
//fmt.Println(v1)
//3.修改
fmt.Println(v1.CanSet())//是否可以进行修改
v2 := reflect.ValueOf(&circle) // 传递指针才能修改
v4:=v2.Elem()// 传递指针才能修改,获取Elem()才能修改
fmt.Println(v4.CanSet()) //true
v4.SetFloat(3.14)
fmt.Println(circle) //3.14
}
operation result:
Reflect : circle.Value = 6.28
Reflect : circle.Type = float64
6.28
6.28
6.28
false
true
3.14
illustrate
- The parameter that needs to be passed in is the pointer of * float64, and then you can use pointer.Elem() to get the pointed Value. Note that it must be a pointer .
- If the incoming parameter is not a pointer but a variable, then
- Get the object corresponding to the original value through Elem and panic directly
- Query whether it can be set through the CanSet method and return false
- newValue.CantSet() indicates whether its value can be reset. If the output is true, it can be modified, otherwise it cannot be modified. After the modification, print it and find that it has really been modified.
- reflect.Value.Elem() means to obtain the reflection object corresponding to the original value, only the original object can be modified, the current reflection object cannot be modified
- That is to say, if you want to modify the reflection type object, its value must be "addressable" [the corresponding pointer is to be passed in, and the reflection object corresponding to the original value must be obtained through the Elem method]
- The nesting of struct or struct is the same judgment processing method
Method calls through reflect.ValueOf
This is considered an advanced usage. Earlier we only talked about the usage of several reflections on types and variables, including how to get their values, their types, and how to reset new values. But in engineering applications, another commonly used and advanced usage is to call methods [functions] through reflect. For example, when we are doing a framework project, we need to be able to extend the method at will, or the user can customize the method, so how can we extend it so that the user can customize it? The key point is that the user's custom method is unknown, so we can use reflect to get it done.
sample code
package main
import (
"fmt"
"reflect"
)
type Person struct {
Name string
Age int
Sex string
}
func (p Person) Say(msg string){
fmt.Println("hello,",msg)
}
func (p Person) PrintInfo(){
fmt.Printf("姓名：%s，年龄：%d，性别：%s\n",p.Name,p.Age,p.Sex)
}
func (p Person) Test(i,j int,s string){
fmt.Println(i,j,s)
}
func main() {
/*
通过反射来进行方法的调用
思路：
step1：接口变量-->对象反射对象：Value
step2：获取对应的方法对象：MethodByName()
step3：将方法对象进行调用：Call()
*/
p1 := Person{
"Ruby",20,"男"}
value :=reflect.ValueOf(p1)
fmt.Printf("kind : %s, type:%s\n",value.Kind(),value.Type()) //kind : struct, type:main.Person
methodValue1 :=value.MethodByName("PrintInfo")
fmt.Printf("kind:%s,type:%s\n",methodValue1.Kind(),methodValue1.Type()) //kind:func,type:func()
//没有参数，进行调用
methodValue1.Call(nil) //没有参数，直接写nil
args1 := make([]reflect.Value,0) //或者创建一个空的切片也可以
methodValue1.Call(args1)
methodValue2:=value.MethodByName("Say")
fmt.Printf("kind:%s, type:%s\n",methodValue2.Kind(),methodValue2.Type()) //kind:func, type:func(string)
args2:=[]reflect.Value{
reflect.ValueOf("反射机制")}
methodValue2.Call(args2)
methodValue3:=value.MethodByName("Test")
fmt.Printf("kind:%s,type:%s\n",methodValue3.Kind(),methodValue3.Type())//kind:func,type:func(int, int, string)
args3:=[]reflect.Value{
reflect.ValueOf(100),reflect.ValueOf(200),reflect.ValueOf("Hello World")}
methodValue3.Call(args3)
}
operation result:
kind : struct, type:main.Person
kind:func,type:func()
姓名：Ruby，年龄：20，性别：男
姓名：Ruby，年龄：20，性别：男
kind:func, type:func(string)
hello, 反射机制
kind:func,type:func(int, int, string)
100 200 Hello World
Through reflection, call the function
package main
import (
"fmt"
"reflect"
"strconv"
)
func main() {
//函数的反射
/*
思路：函数也是看做接口变量类型
step1：函数--->反射对象，Value
step2：kind-->func
step3：call()
*/
f1 := fun1
value := reflect.ValueOf(f1)
fmt.Printf("kind:%s, type :%s\n", value.Kind(), value.Type()) //kind:func, type :func()
value2 := reflect.ValueOf(fun2)
value3 := reflect.ValueOf(fun3)
fmt.Printf("kind:%s,type:%s\n", value2.Kind(), value2.Type()) //kind:func,type:func(int, string)
fmt.Printf("kind:%s,type:%s\n", value3.Kind(), value3.Type()) //kind:func,type:func(int, string) string
//通过反射调用函数
value.Call(nil)
value2.Call([]reflect.Value{
reflect.ValueOf(1000), reflect.ValueOf("张三")})
resultValue := value3.Call([]reflect.Value{
reflect.ValueOf(2000), reflect.ValueOf("Ruby")})
fmt.Printf("%T\n", resultValue) //[]reflect.Value
fmt.Println(len(resultValue)) //1
fmt.Printf("kind:%s,type:%s\n", resultValue[0].Kind(), resultValue[0].Type()) //kind:string,type:string
s := resultValue[0].Interface().(string)
fmt.Println(s)
fmt.Printf("%T\n", s)
}
func fun1() {
fmt.Println("我是函数fun1(),无参的...")
}
func fun2(i int, s string) {
fmt.Println("我是函数fun2(),有参的。。", i, s)
}
func fun3(i int, s string) string {
fmt.Println("我是函数fun3()，有参的，也有返回值。。", i, s)
return s + strconv.Itoa(i)
}
operation result:
kind:func, type :func()
kind:func,type:func(int, string)
kind:func,type:func(int, string) string
我是函数fun1(),无参的...
我是函数fun2(),有参的。。 1000 张三
我是函数fun3()，有参的，也有返回值。。 2000 Ruby
[]reflect.Value
1
kind:string,type:string
Ruby2000
string
structure
anonymous struct
package main
import (
"fmt"
"reflect"
)
type Animal struct {
Name string
Age int
}
type Cat struct {
Animal
Color string
}
// 获取匿名字段
func main() {
c1 := Cat{
Animal{
"猫咪", 1}, "白色"}
t1 := reflect.TypeOf(c1)
for i := 0; i < t1.NumField(); i++ {
fmt.Println(t1.Field(i))
/*
{Animal main.Animal 0 [0] true}
{Color string 24 [1] false}
*/
}
// FiledByIndex()的参数是一个切片，第一个数是Animal字段，第二个参数是Animal的第一个字段
f1 := t1.FieldByIndex([]int{
0, 0})
f2 := t1.FieldByIndex([]int{
0, 1})
fmt.Println(f1) //{Name string 0 [0] false}
fmt.Println(f2) //{Age int 16 [1] false}
v1 := reflect.ValueOf(c1)
fmt.Println(v1.Field(0)) //{猫咪 1}
fmt.Println(v1.FieldByIndex([]int{
0, 0})) //猫咪
}
operation result:
{Animal main.Animal 0 [0] true}
{Color string 24 [1] false}
{Name string 0 [0] false}
{Age int 16 [1] false}
{猫咪 1}
猫咪
Through reflection, modify the data of the structure
Sample code:
package main
import (
"reflect"
"fmt"
)
type Student struct {
Name string
Age int
School string
}
func main() {
/*
修改内容
*/
s1:= Student{
"王二狗",18,"清华大学"}
v1 := reflect.ValueOf(&s1)
if v1.Kind() ==reflect.Ptr && v1.Elem().CanSet(){
v1 = v1.Elem()
fmt.Println("可以修改。。")
}
f1:=v1.FieldByName("Name")
fmt.Println(f1.CanSet())
f1.SetString("王三狗")
f2:=v1.FieldByName("Age")
fmt.Println(f2.CanSet())
f2.SetInt(20)
fmt.Println(s1)
}
operation result:
可以修改。。
true
true
{王三狗 20 清华大学}