区块链信用证项目：golang内存泄漏，goroutine泄漏排查

一、pprof的使用

pprof的启动及操作

import _ "net/http/pprof"
go func() {
        http.ListenAndServe("0.0.0.0:11181", nil)
}()

然后通过网页打开即可：http://192.168.9.78:11181/debug/pprof/
pprof的一些操作：
go tool pprof -inuse_space http://192.168.9.78:11181/debug/pprof/heap

pprof信息的解读：

/debug/pprof/goroutine:

goroutine profile: total 63
4 @ 0x4382fa 0x433417 0x432a59 0x539888 0x5398f4 0x53b047 0x54ee20 0x643e08 0x644374 0x64807d 0x57dcf2 0x47aba9 0x47ad18 0xafc78b 0xafd0e4 0x119c18f 0x118eb7d 0x465691
#   0x432a58    net.runtime_pollWait+0x58                               /usr/local/go/src/runtime/netpoll.go:164
#   0x539887    net.(*pollDesc).wait+0x37                               /usr/local/go/src/net/fd_poll_runtime.go:75
#   0x5398f3    net.(*pollDesc).waitRead+0x33                               /usr/local/go/src/net/fd_poll_runtime.go:80
#   0x53b046    net.(*netFD).Read+0x1b6                                 /usr/local/go/src/net/fd_unix.go:250
#   0x54ee1f    net.(*conn).Read+0x6f                                   /usr/local/go/src/net/net.go:181
#   0x643e07    crypto/tls.(*block).readFromUntil+0x97                          /usr/local/go/src/crypto/tls/conn.go:488
#   0x644373    crypto/tls.(*Conn).readRecord+0xc3                          /usr/local/go/src/crypto/tls/conn.go:590
#   0x64807c    crypto/tls.(*Conn).Read+0x11c                               /usr/local/go/src/crypto/tls/conn.go:1134
#   0x57dcf1    bufio.(*Reader).Read+0x311                              /usr/local/go/src/bufio/bufio.go:213
#   0x47aba8    io.ReadAtLeast+0xa8                                 /usr/local/go/src/io/io.go:307
#   0x47ad17    io.ReadFull+0x57                                    /usr/local/go/src/io/io.go:325
#   0xafc78a    Masami/backEnd/vendor/golang.org/x/net/http2.readFrameHeader+0x7a           /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:237
#   0xafd0e3    Masami/backEnd/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3           /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:492
#   0x119c18e   Masami/backEnd/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e     /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http_util.go:608
#   0x118eb7c   Masami/backEnd/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xcc   /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http2_client.go:1115

4 @ 0x4382fa 0x447704 0x44636c 0x118fa7a 0x465691
#   0x118fa79   Masami/backEnd/vendor/google.golang.org/grpc/transport.(*http2Client).controller+0x659  /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http2_client.go:1188

解读：总数，该协程的数量@协程运行的位置

/debug/pprof/heap:

heap profile: 20: 1547872 [155297: 2755898608] @ heap/1048576
1: 1384448 [2: 2768896] @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691
#   0x775a9e    runtime/pprof.writeHeap+0x8e        /usr/local/go/src/runtime/pprof/pprof.go:489
#   0x773b44    runtime/pprof.(*Profile).WriteTo+0x3b4  /usr/local/go/src/runtime/pprof/pprof.go:302
#   0xa10570    net/http/pprof.handler.ServeHTTP+0x1d0  /usr/local/go/src/net/http/pprof/pprof.go:209
#   0xa10882    net/http/pprof.Index+0x1e2      /usr/local/go/src/net/http/pprof/pprof.go:221
#   0x6e8b03    net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
#   0x6e9f3f    net/http.(*ServeMux).ServeHTTP+0x12f    /usr/local/go/src/net/http/server.go:2238
#   0x6eb301    net/http.serverHandler.ServeHTTP+0x91   /usr/local/go/src/net/http/server.go:2568
#   0x6e7671    net/http.(*conn).serve+0x611        /usr/local/go/src/net/http/server.go:1825

1: 155648 [1: 155648] @ 0x609d3f 0x47957e 0x60a3bb 0x608925 0x61ab8e 0x619ada 0x933fe8 0x934590 0x952e9e 0x950540 0x958d10 0x167b977 0x167b10b 0x17b61bc 0x1792cb5 0x17d8994 0x465691
#   0x609d3e    crypto/elliptic.initTable+0x3e                                      /usr/local/go/src/crypto/elliptic/p256_amd64.go:382
#   0x47957d    sync.(*Once).Do+0xbd                                            /usr/local/go/src/sync/once.go:44
#   0x60a3ba    crypto/elliptic.(*p256Point).p256BaseMult+0x4a                              /usr/local/go/src/crypto/elliptic/p256_amd64.go:426
#   0x608924    crypto/elliptic.p256Curve.ScalarBaseMult+0xc4                               /usr/local/go/src/crypto/elliptic/p256_amd64.go:240
#   0x61ab8d    crypto/x509.parseECPrivateKey+0x41d                                 /usr/local/go/src/crypto/x509/sec1.go:102
#   0x619ad9    crypto/x509.ParsePKCS8PrivateKey+0x369                                  /usr/local/go/src/crypto/x509/pkcs8.go:45
#   0x933fe7    Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils.DERToPrivateKey+0x77            /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils/keys.go:192
#   0x93458f    Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils.PEMtoPrivateKey+0x38f           /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils/keys.go:237
#   0x952e9d    Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw.(*fileBasedKeyStore).loadPrivateKey+0x3ad  /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw/fileks.go:338
#   0x95053f    Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw.(*fileBasedKeyStore).GetKey+0x4bf      /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw/fileks.go:134
#   0x958d0f    Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw.(*impl).GetKey+0x6f            /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw/impl.go:255
#   0x167b976   Masami/backEnd/vendor/ILIOS/core/fabricsdk.getDefaultImplPreEnrolledUser+0x7d6              /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/fabricsdk/sdkutil.go:107
#   0x167b10a   Masami/backEnd/vendor/ILIOS/core/fabricsdk.GetAdmin+0x24a                       /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/fabricsdk/sdkutil.go:49
#   0x17b61bb   Masami/backEnd/vendor/ILIOS/core/blockchain.setupClient+0x16b                       /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainSdkConfig.go:391
#   0x1792cb4   Masami/backEnd/vendor/ILIOS/core/blockchain.setChannel+0x54                     /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1212
#   0x17d8993   Masami/backEnd/vendor/ILIOS/core/blockchain.(*BlockchainChannel).QueryBlock.func1+0xb3          /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1231

.....

# runtime.MemStats
# Alloc = 16831912
# TotalAlloc = 93465363144
# Sys = 111851768
# Lookups = 436597
# Mallocs = 2411037775
# Frees = 2410767219
# HeapAlloc = 16831912
# HeapSys = 30670848
# HeapIdle = 5324800
# HeapInuse = 25346048
# HeapReleased = 0
# HeapObjects = 270556
# Stack = 1179648 / 1179648
# MSpan = 415720 / 425984
# MCache = 4800 / 16384
# BuckHashSys = 2750142
# GCSys = 75923456
# OtherSys = 885306
# NextGC = 22628176
# PauseNs = [251625 102144 1114973 82811 962412 55545 70193 71273 83591 2671351 80293 61109 95033 78908 131701 59840 84397 857675 129996 1180145 60259 104967 54462 92301 63168 87525 3775060 67801 98686 75025 52108 4226429 69942 83824 101308 100086 222931 85903 141632 108922 49959 77062 41757 111464 91770 653235 83308 55181 87047 86229 78250 66959 128160 69158 83498 81554 91212 104690 81473 98758 66789 170286 103240 66697 85881 55563 119912 157109 115442 82078 98494 130530 113911 123343 45100 144668 3457395 111751 76875 100159 87038 87886 90464 118198 3193192 105136 112051 73821 61658 102659 79153 62140 100095 75005 71233 8313802 238423 70598 57186 801672 84199 69673 1850699 352710 372426 80204 91827 88774 105581 168779 117494 3621083 45458 96822 80238 83728 87826 107449 68569 109294 3241219 71098 3377295 76956 3705284 187921 3296415 3589375 128336 125871 126073 118386 84267 150214 68185 78886 180731 65156 102083 57375 83865 84446 1795706 1987823 81895 452550 75960 84030 159927 88024 77141 64582 154597 106656 91543 3260111 101412 63701 102325 71723 104087 67210 119190 74439 73354 126235 63346 101999 78577 76937 100137 60490 57165 149083 87388 248331 115907 82016 101468 97311 89321 114089 242282 103262 180517 88439 94698 5247411 248777 79423 105661 111279 91862 202097 103120 64497 62549 101927 77345 107139 61002 3071591 68166 101282 1082715 73638 893508 116980 121683 149156 65585 78968 522928 51675 4243053 128256 130818 4473130 78633 163665 230193 123923 93318 96058 63285 91195 2865382 3166808 69171 88237 106198 58686 1892996 92479 74520 95618 887626 426498 87927 118181 108641 116303 126501 78906 100257 144567 3517513 101324 106846 123704 82077 2957290 104086 89691 108962 223224]
# NumGC = 13036
# DebugGC = false

意思是：heap profile: 20(正在使用的对象数量): 1547872（正在使用的内存大小：bytes） [155297(分配的对象数量): 2755898608（分配的内存大小）] @ heap/1048576

1: 1384448 [2: 2768896]（与上面的含义一样，分别是：正在使用的对象数量， 正在使用的内存大小；分配的对象数量，分配的内存大小） @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691（栈指针地址）

第二部分意思：是runtime.memstats的统计信息

二、民生基于区块链的信用证项目内存泄漏，goroutine泄漏排查：

背景：

系统在做稳定性测试时发现内存持续很快的增长，初步判断有泄漏风险

排查步骤：

一、打开pprof监测程序

堆情况：

heap profile: 66: 2958232 [282858: 9664223888] @ heap/1048576
1: 1368064 [1: 1368064] @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691
#   0x775a9e    runtime/pprof.writeHeap+0x8e        /usr/local/go/src/runtime/pprof/pprof.go:489
#   0x773b44    runtime/pprof.(*Profile).WriteTo+0x3b4  /usr/local/go/src/runtime/pprof/pprof.go:302
#   0xa10570    net/http/pprof.handler.ServeHTTP+0x1d0  /usr/local/go/src/net/http/pprof/pprof.go:209
#   0xa10882    net/http/pprof.Index+0x1e2      /usr/local/go/src/net/http/pprof/pprof.go:221
#   0x6e8b03    net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
#   0x6e9f3f    net/http.(*ServeMux).ServeHTTP+0x12f    /usr/local/go/src/net/http/server.go:2238
#   0x6eb301    net/http.serverHandler.ServeHTTP+0x91   /usr/local/go/src/net/http/server.go:2568
#   0x6e7671    net/http.(*conn).serve+0x611        /usr/local/go/src/net/http/server.go:1825

1: 1359872 [2: 2719744] @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691
#   0x775a9e    runtime/pprof.writeHeap+0x8e        /usr/local/go/src/runtime/pprof/pprof.go:489
#   0x773b44    runtime/pprof.(*Profile).WriteTo+0x3b4  /usr/local/go/src/runtime/pprof/pprof.go:302
#   0xa10570    net/http/pprof.handler.ServeHTTP+0x1d0  /usr/local/go/src/net/http/pprof/pprof.go:209
#   0xa10882    net/http/pprof.Index+0x1e2      /usr/local/go/src/net/http/pprof/pprof.go:221
#   0x6e8b03    net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
#   0x6e9f3f    net/http.(*ServeMux).ServeHTTP+0x12f    /usr/local/go/src/net/http/server.go:2238
#   0x6eb301    net/http.serverHandler.ServeHTTP+0x91   /usr/local/go/src/net/http/server.go:2568
#   0x6e7671    net/http.(*conn).serve+0x611        /usr/local/go/src/net/http/server.go:1825

1: 90112 [1: 90112] @ 0x413ca6 0x7db566 0x7f0ced 0x7fc74e 0x84dcf3 0x8518d8 0x924a90 0x163ef7b 0x17e0068 0x17e32f4 0x191b259 0x437e6a 0x465691
#   0x7db565    html.init+0x75                              /usr/local/go/src/html/entity.go:16
#   0x7f0cec    html/template.init+0x7c                         /usr/local/go/src/html/template/url.go:106
#   0x7fc74d    Masami/backEnd/vendor/github.com/astaxie/beego/context.init+0xcd    /opt/gopath/src/Masami/backEnd/vendor/github.com/astaxie/beego/context/output.go:349
#   0x84dcf2    Masami/backEnd/vendor/github.com/astaxie/beego.init+0xc2        /opt/gopath/src/Masami/backEnd/vendor/github.com/astaxie/beego/tree.go:581
#   0x8518d7    Masami/backEnd/vendor/ILIOS/common/packager.init+0x77           /opt/gopath/src/Masami/backEnd/vendor/ILIOS/common/packager/packager.go:38
#   0x924a8f    Masami/backEnd/vendor/ILIOS/common.init+0x4f                /opt/gopath/src/Masami/backEnd/vendor/ILIOS/common/utils.go:82
#   0x163ef7a   Masami/backEnd/vendor/ILIOS/core.init+0x4a              /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/zkInstance.go:265
#   0x17e0067   Masami/backEnd/vendor/ILIOS/core/blockchain.init+0x47           /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/util.go:366
#   0x17e32f3   Masami/backEnd/vendor/ILIOS/ilioslaunch.init+0x43           /opt/gopath/src/Masami/backEnd/vendor/ILIOS/ilioslaunch/launch.go:14
#   0x191b258   main.init+0x48                              /opt/gopath/src/Masami/backEnd/main.go:47
#   0x437e69    runtime.main+0x1c9                          /usr/local/go/src/runtime/proc.go:173

22: 70400 [65: 208000] @ 0x110b612 0x110eff7 0x110c6ff 0x110ff0c 0x110c6ff 0x110c50e 0x110c190 0x11cb597 0x110cf0d 0x110c50e 0x11af4a9 0x11c34f7 0x11b21bf 0x11bcce2 0x11cc442 0x1665d0c 0x465691
#   0x110b611   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes+0x171                       /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:298
#   0x110eff6   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_slice_slice_byte+0x36                  /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:802
#   0x110c6fe   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
#   0x110ff0b   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_struct_message+0x13b                   /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:920
#   0x110c6fe   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
#   0x110c50d   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
#   0x110c18f   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeMessage+0xbf                     /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:421
#   0x11cb596   Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer._Event_OneofUnmarshaler+0x216                   /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:432
#   0x110cf0c   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x9cc                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:508
#   0x110c50d   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
#   0x11af4a8   Masami/backEnd/vendor/google.golang.org/grpc.protoCodec.Unmarshal+0x118                             /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/codec.go:85
#   0x11c34f6   Masami/backEnd/vendor/google.golang.org/grpc.(*protoCodec).Unmarshal+0x76                           <autogenerated>:35
#   0x11b21be   Masami/backEnd/vendor/google.golang.org/grpc.recv+0x16e                                     /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/rpc_util.go:382
#   0x11bcce1   Masami/backEnd/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1                           /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/stream.go:398
#   0x11cc441   Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer.(*eventsChatClient).Recv+0x61                   /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:559
#   0x1665d0b   Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events/consumer.(*eventsClient).processEvents+0xab /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events/consumer/consumer.go:236

19: 19456 [62: 63488] @ 0x110b612 0x110eff7 0x110c6ff 0x110ff0c 0x110c6ff 0x110c50e 0x110c190 0x11cb597 0x110cf0d 0x110c50e 0x11af4a9 0x11c34f7 0x11b21bf 0x11bcce2 0x11cc442 0x1665d0c 0x465691
#   0x110b611   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes+0x171                       /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:298
#   0x110eff6   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_slice_slice_byte+0x36                  /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:802
#   0x110c6fe   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
#   0x110ff0b   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_struct_message+0x13b                   /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:920
#   0x110c6fe   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
#   0x110c50d   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
#   0x110c18f   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeMessage+0xbf                     /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:421
#   0x11cb596   Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer._Event_OneofUnmarshaler+0x216                   /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:432
#   0x110cf0c   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x9cc                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:508
#   0x110c50d   Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd                        /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
#   0x11af4a8   Masami/backEnd/vendor/google.golang.org/grpc.protoCodec.Unmarshal+0x118                             /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/codec.go:85
#   0x11c34f6   Masami/backEnd/vendor/google.golang.org/grpc.(*protoCodec).Unmarshal+0x76                           <autogenerated>:35
#   0x11b21be   Masami/backEnd/vendor/google.golang.org/grpc.recv+0x16e                                     /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/rpc_util.go:382
#   0x11bcce1   Masami/backEnd/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1                           /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/stream.go:398
#   0x11cc441   Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer.(*eventsChatClient).Recv+0x61                   /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:559
#   0x1665d0b   Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events

协程情况：

goroutine profile: total 4023
3964 @ 0x4382fa 0x4383de 0x40f59d 0x40f2ad 0x17d90cc 0x16729b7 0x465691
#   0x17d90cb   Masami/backEnd/vendor/ILIOS/core/blockchain.(*BlockchainChannel).RegisterBlockEvent.func1+0x1cb             /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1491
#   0x16729b6   Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events.(*EventHub).Recv.func1+0x336    /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events/eventhub.go:280

4 @ 0x4382fa 0x433417 0x432a59 0x539888 0x5398f4 0x53b047 0x54ee20 0x643e08 0x644374 0x64807d 0x57dcf2 0x47aba9 0x47ad18 0xafc78b 0xafd0e4 0x119c18f 0x118eb7d 0x465691
#   0x432a58    net.runtime_pollWait+0x58                               /usr/local/go/src/runtime/netpoll.go:164
#   0x539887    net.(*pollDesc).wait+0x37                               /usr/local/go/src/net/fd_poll_runtime.go:75
#   0x5398f3    net.(*pollDesc).waitRead+0x33                               /usr/local/go/src/net/fd_poll_runtime.go:80
#   0x53b046    net.(*netFD).Read+0x1b6                                 /usr/local/go/src/net/fd_unix.go:250
#   0x54ee1f    net.(*conn).Read+0x6f                                   /usr/local/go/src/net/net.go:181
#   0x643e07    crypto/tls.(*block).readFromUntil+0x97                          /usr/local/go/src/crypto/tls/conn.go:488
#   0x644373    crypto/tls.(*Conn).readRecord+0xc3                          /usr/local/go/src/crypto/tls/conn.go:590
#   0x64807c    crypto/tls.(*Conn).Read+0x11c                               /usr/local/go/src/crypto/tls/conn.go:1134
#   0x57dcf1    bufio.(*Reader).Read+0x311                              /usr/local/go/src/bufio/bufio.go:213
#   0x47aba8    io.ReadAtLeast+0xa8                                 /usr/local/go/src/io/io.go:307
#   0x47ad17    io.ReadFull+0x57                                    /usr/local/go/src/io/io.go:325
#   0xafc78a    Masami/backEnd/vendor/golang.org/x/net/http2.readFrameHeader+0x7a           /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:237
#   0xafd0e3    Masami/backEnd/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3           /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:492
#   0x119c18e   Masami/backEnd/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e     /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http_util.go:608
#   0x118eb7c   Masami/backEnd/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xcc   /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http2_client.go:1115

内存消耗图：

(pprof) top
31857.57kB of 32881.59kB total (96.89%)
Dropped 1095 nodes (cum <= 164.41kB)
Showing top 10 nodes out of 112 (cum >= 512.02kB)
      flat  flat%   sum%        cum   cum%
22576.68kB 68.66% 68.66% 22576.68kB 68.66%  Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes
 2048.81kB  6.23% 74.89%  2048.81kB  6.23%  runtime.malg
 2048.12kB  6.23% 81.12% 24112.79kB 73.33%  Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_slice_slice_byte
 1536.08kB  4.67% 85.79%  1536.08kB  4.67%  reflect.unsafe_New
 1078.31kB  3.28% 89.07%  1078.31kB  3.28%  runtime.makemap
  520.04kB  1.58% 90.65%   520.04kB  1.58%  Masami/backEnd/vendor/github.com/go-sql-driver/mysql.(*buffer).fill
  513.31kB  1.56% 92.21%   513.31kB  1.56%  regexp/syntax.(*compiler).compile
  512.16kB  1.56% 93.77%   512.16kB  1.56%  Masami/backEnd/vendor/golang.org/x/net/trace.(*histogram).Add
  512.02kB  1.56% 95.33%   512.02kB  1.56%  Masami/backEnd/vendor/google.golang.org/grpc.newClientStream
  512.02kB  1.56% 96.89%   512.02kB  1.56%  runtime.rawstringtmp

上述信息可知：

• Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes消耗了大部分内存
• /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1491处有协程泄漏

原因分析：

• 一般来讲，有协程泄漏的地方肯定会导致内存泄漏，之前遇到过grpc接收区块时，资源不释放，总之我们首先解决协程泄漏的问题。

review代码：

/opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1491:

//RegisterBlockEvent 通用的block监听方法，不会过滤channelid
func (channel *BlockchainChannel) RegisterBlockEvent(peername string, filterByChannelID bool) (chan *common.Block, error) {
    logs.Debug("gggRegisterBlockEvent\n")
    client, _, err := setupClient(channel.UserId, peername)
    if err != nil {
        return nil, err
    }
    eventHub, err := events.NewEventHub(client)
    if err != nil {
        return nil, err
    }
    peerinfo, err := getPeerKeystorePathAndUrl(channel.UserId, peername)
    if err != nil {
        return nil, err
    }
    eventHub.SetPeerAddr(peerinfo.EventUrl, string(peerinfo.TLSCert.Content), peerinfo.CommonName)
    err = eventHub.Connect()
    if err != nil {
        return nil, err
    }
    blockchan := make(chan *common.Block)
    eventHub.RegisterBlockEvent(func(block *common.Block) {
        logs.Info("ggggReceived callback on block event")
        if filterByChannelID == false {
            blockchan <- block
            logs.Debug("ggggfilterByChannelID\n")
        } else {
            for _, tdata := range block.Data.Data {
                channelid, err := getChannelID(tdata)
                if err == nil && channelid == channel.ChannelID {
                    blockchan <- block
                    logs.Debug("gggggChannelID:%s\n", channelid)
                    break
                }
            }
        }
    })
    return blockchan, nil
}

func GetMonitor(namespace string, channel string, peer string) {
    logs.Debug("ggggGetMonitor\n")
    for {
        select {
        case block := <-common.BlockEventhub[namespace][channel][peer].Block:
            logs.Debug("AAA channel:%s,%s,%s\n", namespace, channel, peer)
            blocknum, transnum, ectivetransnum, nowtime := DesBlockMonitor(block)
            logs.Debug("BBB channel: ", channel, "peer: ", peer, "blocknum: ", blocknum, "transnum: ", transnum, "ectivetransnum: ", ectivetransnum, "nowtime: ", nowtime)
            // logs.Debug("AAA nowBlowNum: ", common.BlockEventhub[namespace][channel][peer].BlockNum)
            if blocknum > common.BlockEventhub[namespace][channel][peer].BlockNum {
                common.BlockEventhub[namespace][channel][peer].BlockNum = blocknum
                common.UserChannelTps[namespace][channel].ChannelLock.Lock()
                index := len(common.UserChannelTps[namespace][channel].TPSTimeStamp) - 1
                common.UserChannelTps[namespace][channel].TPSList[index] += transnum
                common.UserChannelTps[namespace][channel].BlockMakeList[index] += 1
                common.UserChannelTps[namespace][channel].ChannelLock.Unlock()
            }
        }
    }

}

func RegisterEventInit(namespace string, channelname string, peername []string) {
    if _, ok := common.BlockEventhub[namespace]; !ok {
        common.BlockEventhub[namespace] = make(map[string](map[string]*common.ChannelMonitor))
    }
    if _, ok := common.BlockEventhub[namespace][channelname]; !ok {
        common.BlockEventhub[namespace][channelname] = make(map[string]*common.ChannelMonitor)
    }
    logs.Debug("gggggpeername:%s\n", peername)
    for i, peer := range peername {
        logs.Debug("ggggi=%d,peer:%s\n", i, peer)
        go func() {
            logs.Debug("ggggnamespace: ", namespace, channelname, peer)
            address, err := GetPeerEventAddress(namespace, peer)
            if err != nil {
                logs.Error("RegisterEventInit GetPeerEventAddress Error: ", err)
                return
            }
            showlog := false
            ticker := time.NewTicker(2 * time.Second)
            registerflag := false
            for _ = range ticker.C {
                if err := tcpTimeout(address); err != nil {
                    logs.Debug("ggggtcptimeout,addr:%s, err:%s\n", address, err)
                    showlog = false
                    logs.Warning("*************EVENT TRY TO CONNECT user %s %v***********", namespace, err)
                    channel := blockchain.NewBlockchainChannel(namespace, channelname)
                    blockchan, err := channel.RegisterBlockEvent(peer, true)
                    if err != nil {
                        logs.Error("RegisterEventInit Error: ", err)

                    } else {
                        common.BlockEventhubLock.Lock()
                        if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
                            common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
                            common.BlockEventhub[namespace][channelname][peer].Block = blockchan
                            go GetMonitor(namespace, channelname, peer)
                        }
                        common.BlockEventhubLock.Unlock()
                    }
                } else {
                    if !registerflag {
                        registerflag = true
                        channel := blockchain.NewBlockchainChannel(namespace, channelname)
                        blockchan, _ := channel.RegisterBlockEvent(peer, true)
                        if err != nil {
                            logs.Error("RegisterEventInit Error: ", err)
                        } else {
                            logs.Debug("gggggregblockevent\n")
                            common.BlockEventhubLock.Lock()
                            if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
                                common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
                                common.BlockEventhub[namespace][channelname][peer].Block = blockchan
                                go GetMonitor(namespace, channelname, peer)
                            }
                            common.BlockEventhubLock.Unlock()
                        }
                    }
                    if showlog == false {
                        logs.Debug("*************EVENT HAS CONNECTED user %s ***********", namespace)
                        showlog = true
                    }
                }
            }
        }()
    }
}

func GetBlockFromFabricInit(namespace string) {

    channels, _ := GetAllChannels(namespace)
    for _, channel := range channels {
        InitUserChannelTps(namespace, channel.ChannelName)
        // InitUserChannelTpsNode(namespace, channel)
        GetBlockFromFabricByChannel(namespace, channel)
        // go CalTPS(namespace, channel)

        //register eventhub
        if channel.UserNodename != "" && channel.Status == common.CHANNELJOIN {
            comp, err := GetNameSpaceOrgNameByNamespace(namespace)
            if err != nil {
                logs.Error("Get company by namespace error: ", err)
                continue
            }
            nodes, err := GetPeerByNamespaceAndChannelAndComp(namespace, channel.ChannelName, comp)
            if err != nil {
                logs.Error("GetPeerByNamespaceAndChannelAndComp error: ", err)
                continue
            }
            nodeStrs := []string{}
            for _, node := range nodes {
                nodeStrs = append(nodeStrs, node.Name)
            }
            logs.Debug("ggggGetBlockFromFabricInitRegisterEventInit:%s,%s,%s\n", namespace, channel.ChannelName, nodeStrs)
            RegisterEventInit(namespace, channel.ChannelName, nodeStrs)
            //ADD
            go QueryChaincodeListInvoke(namespace, channel.ChannelName)
        }

    }

}

可能原因：

• blockchan被阻塞：
  
  • getmonitor接收的不完全，接收区块处理过程可能被锁住（注释掉处理区块的操作，现象不变，所以排除该因）
  • 还有blockchain通道的block没有被接收
  • 有全局变量导致协程无法释放（协程可以到达return，应该也不会由此引起）
  • 协程没有返回，一直运行（通过加日志发现是可以到达return的，所以排除该因）
    于是加了一些打印，查看可能出现问题的地方

日志信息：

2018/06/07 10:52:10 [I] [eventhub.go:280] ggggReceived callback on block event
2018/06/07 10:52:10 [D] [eventhub.go:280] gggggChannelID:mychannel

2018/06/07 10:52:10 [D] [asm_amd64.s:2197] after send chan

2018/06/07 10:52:10 [D] [asm_amd64.s:2197] before return

2018/06/07 10:52:10 [D] [asm_amd64.s:2197] AAA channel:4ddc1373-e73b-4679-8560-c3d26b4c64ff,mychannel,peer-1-yunphant1

2018/06/07 10:52:10 [D] [asm_amd64.s:2197] BBB channel:  mychannel peer:  peer-1-yunphant1 blocknum:  84920 transnum:  1 ectivetransnum:  1 nowtime:  1528368730
2018/06/07 10:52:10 [D] [asm_amd64.s:2197] recv count2, time:2018-06-07 10:52:10.560654537 +0000 UTC, Recv blockEvent:&{header:<6:73 0:"\x01K\xb8" } 4:11890 157618:/* unknown wire type 6 */ 145:40 747:/* unexpected EOF 

2018/06/07 10:52:08 [D] [browserbass.go:43] gggggpeername:[peer-0-yunphant1 peer-1-yunphant1]

2018/06/07 10:52:08 [D] [browserbass.go:43] ggggi=0,peer:peer-0-yunphant1

2018/06/07 10:52:08 [D] [browserbass.go:43] ggggi=1,peer:peer-1-yunphant1

2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ================QueryBaasChaincodeEvent BEGIN=================
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ggggnamespace:  4ddc1373-e73b-4679-8560-c3d26b4c64ff mychannel peer-1-yunphant1
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ============QueryChaincodeListInvoke Begin=========== mychannel
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ====== Begin QueryChannelListBaaS ======
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ggggnamespace:  4ddc1373-e73b-4679-8560-c3d26b4c64ff mychannel peer-1-yunphant1

现象总结：

• 每两秒产生一个协程
• 接收区块的协程也就是getmonitor函数，每两秒才接收一个
• grpc.Recv()要比每两秒一个区块多
• 创建blockchan时，两个peername居然一样

 if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
    common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
    common.BlockEventhub[namespace][channelname][peer].Block = blockchan
    go GetMonitor(namespace, channelname, peer)
}

这里会对同样namespace,peername的map进行判断，第二次时不会对

common.BlockEventhub[namespace][channelname][peer].Block = blockchan

重新赋值，导致blockchan的指针地址不一致，也没有起GetMonitor()，第一次创建的getmonitor没法接受第二次创建的blockchan里的block(因为blockchan是不一致的) ,所以会造成blockchan阻塞。

其实这段代码逻辑上也是有风险的，最好这样写，如果相同namesapce和peer,就不要去registorBlockEvent：

if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
//registBlockEvent的操作也应该放在这个判断里面，不然造成前后不一致，导致内存泄漏，系统崩溃的问题
    channel := blockchain.NewBlockchainChannel(namespace, channelname)
    blockchan, _ := channel.RegisterBlockEvent(peer, true)

    common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
    common.BlockEventhub[namespace][channelname][peer].Block = blockchan
    go GetMonitor(namespace, channelname, peer)
}

解决方案：

• 闭包函数，采用传参方式
• registorBlockevent操作放在判断里面

for i, peer := range peername {
    logs.Debug("ggggi=%d,peer:%s\n", i, peer)
    ...

    go func(peer string) {
        if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
        //registBlockEvent的操作也应该放在这个判断里面，不然造成前后不一致，导致内存泄漏，系统崩溃的问题
        channel := blockchain.NewBlockchainChannel(namespace, channelname)
        blockchan, _ := channel.RegisterBlockEvent(peer, true)

        common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
        common.BlockEventhub[namespace][channelname][peer].Block = blockchan
        go GetMonitor(namespace, channelname, peer)
    }

    ...
}(peer)

for i,peer := range peername, 所有的goroutine都会并发执行。for range循环也许在第一个最多第二个goroutine还在运行的时候就运行完了，peer变量只会有最后一次循环时候的值。也就是说即使不是全部的goroutine也是大部分的goroutine会处理这些变量的相同的值。
我们可以声明可以接收参数的匿名函数，这些类型的闭包问题也就引刃而解了。

这也解释了两秒产生一个协程，因为两个节点中只有最后一个peer的blockchan有接收，第一的peer的block被阻塞了，也就有了每两秒产生一个goroutine了，因为区块时每两秒产生一个。

这样的话，保证了每个通道的block都可被接收，不会有协程阻塞。泄漏问题也得以解决，内存稳定。

PS:

查看进程内存情况

a、top
b、pmap -d 12345
c、ps -e -o ‘pid,comm,args,pcpu,rsz,vsz,stime,user,uid’ 其中rsz是是实际内存
ps -e -o ‘pid,comm,args,pcpu,rsz,vsz,stime,user,uid’ | grep backEnd | sort -nrk5

golang闭包函数

闭包的字面定义：闭包是由函数及其相关引用环境组合而成的实体(即：闭包=函数+引用环境)。闭包函数可以直接引用外层代码定义的变量，闭包函数里面引用的是变量的地址，也就是说闭包函数会改变外面变量的值，当goroutine被调度时，改地址的值才会被传递给goroutine 函数。

for i, peer := range peername {
        logs.Debug("ggggi=%d,peer:%s\n", i, peer)
        go func(peer string) {
        dosomething(peer)
        }(peer)

当这种for i,peer := range peername的程序时需要注意，如果想传入遍历peername的peer的值的话需要通过接受参数的形式;
如果代码：

for i, peer := range peername {
        logs.Debug("ggggi=%d,peer:%s\n", i, peer)
        go func() {
        dosomething(peer)
        }()

那么peer的基本上是peername里的最后一个值，因为go都是并发执行，可能第一个go func()还没运行完，range peername就运行完了，那么后面的go func(){}的peer值就是最后一个。