I got this warning and ‘error’:
-06-08T18:30:51Z WARN servers service takes long to shutdown {"Process": "storagenode", "name": "server"}
2024-06-08T18:30:51Z INFO servers slow shutdown {"Process": "storagenode", "stack": "goroutine 1060 [running]:\nstorj.io/storj/private/lifecycle.(*Group).logStackTrace.func1()\n\t/go/src/storj.io/storj/private/lifecycle/group.go:107 +0x71\nsync.(*Once).doSlow(0xc094bfc750?, 0xc095f4e200?)\n\t/usr/local/go/src/sync/once.go:74 +0xbf\nsync.(*Once).Do(...)\n\t/usr/local/go/src/sync/once.go:65\nstorj.io/storj/private/lifecycle.(*Group).logStackTrace(0xc000491eb8?)\n\t/go/src/storj.io/storj/private/lifecycle/group.go:104 +0x3c\nstorj.io/storj/private/lifecycle.(*Group).Run.func1({0x157f300?, 0xc00036ac60?})\n\t/go/src/storj.io/storj/private/lifecycle/group.go:77 +0x2a5\nruntime/pprof.Do({0x157f5b0?, 0xc0002cda40?}, {{0xc0000b5740?, 0xc000491fd0?, 0xaf2c05?}}, 0xc000459440)\n\t/usr/local/go/src/runtime/pprof/runtime.go:51 +0x9d\ncreated by storj.io/storj/private/lifecycle.(*Group).Run in goroutine 1\n\t/go/src/storj.io/storj/private/lifecycle/group.go:64 +0x509\n\ngoroutine 1 [semacquire, 1246 minutes]:\nsync.runtime_Semacquire(0x1?)\n\t/usr/local/go/src/runtime/sema.go:62 +0x25\nsync.(*WaitGroup).Wait(0xc00016fbc0?)\n\t/usr/local/go/src/sync/waitgroup.go:116 +0x48\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000459340)\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:56 +0x25\nstorj.io/storj/storagenode.(*Peer).Run(0xc000166840, {0x157f338, 0xc000354e10})\n\t/go/src/storj.io/storj/storagenode/peer.go:969 +0x42b\nmain.cmdRun(0x7fab7822a460?, 0xc000220d80)\n\t/go/src/storj.io/storj/cmd/storagenode/cmd_run.go:123 +0xdc5\nmain.newRunCmd.func1(0x1071260?, {0xc000216150?, 0xc000005200?, 0x447440?})\n\t/go/src/storj.io/storj/cmd/storagenode/cmd_run.go:33 +0x17\nstorj.io/common/process.cleanup.func1.4({0x157f5b0?, 0xc0002cc960})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/process/exec_conf.go:393 +0x149\nstorj.io/common/process.cleanup.func1(0xc000005200, {0xc00022d5e0, 0x0, 0xe})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/process/exec_conf.go:411 +0x1c88\ngithub.com/spf13/cobra.(*Command).execute(0xc000005200, {0xc00022d500, 0xe, 0xe})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.8.0/command.go:983 +0xabc\ngithub.com/spf13/cobra.(*Command).ExecuteC(0xc000004300)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.8.0/command.go:1115 +0x3ff\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.8.0/command.go:1039\nstorj.io/common/process.ExecWithCustomOptions(0xc000004300, {0x1, 0x1, 0x1, 0x0, 0x14694d0, 0xc00021bfa0})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/process/exec_conf.go:112 +0x1c9\nmain.main()\n\t/go/src/storj.io/storj/cmd/storagenode/main.go:34 +0x2bf\n\ngoroutine 6 [select]:\nstorj.io/monkit-jaeger.(*ThriftCollector).Run(0xc00017f050, {0x157f338, 0xc00022e730})\n\t/go/pkg/mod/storj.io/monkit-jaeger@v0.0.0-20240221095020-52b0792fa6cd/thrift.go:174 +0x2cf\nstorj.io/common/process.cleanup.func1.2()\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/process/exec_conf.go:351 +0x1f\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78 +0x56\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:75 +0x96\n\ngoroutine 7 [semacquire, 1246 minutes]:\nsync.runtime_Semacquire(0xc000013218?)\n\t/usr/local/go/src/runtime/sema.go:62 +0x25\nsync.(*WaitGroup).Wait(0x11270a0?)\n\t/usr/local/go/src/sync/waitgroup.go:116 +0x48\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc00021c800)\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:56 +0x25\nstorj.io/common/debug.(*Server).Run(0xc000002000, {0x157f2c8, 0x1db5f80})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/debug/server.go:203 +0x33c\nstorj.io/common/process.initDebug.func1()\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/process/debug.go:40 +0xf4\ncreated by storj.io/common/process.initDebug in goroutine 1\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/process/debug.go:37 +0xf0\n\ngoroutine 1122 [IO wait]:\ninternal/poll.runtime_pollWait(0x7fab78184bf8, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:343 +0x85\ninternal/poll.(*pollDesc).wait(0xc00040e980?, 0xc0c7675d90?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Accept(0xc00040e980)\n\t/usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac\nnet.(*netFD).accept(0xc00040e980)\n\t/usr/local/go/src/net/fd_unix.go:172 +0x29\nnet.(*TCPListener).accept(0xc0000502e0)\n\t/usr/local/go/src/net/tcpsock_posix.go:152 +0x1e\nnet.(*TCPListener).AcceptTCP(0xc0000502e0)\n\t/usr/local/go/src/net/tcpsock.go:302 +0x30\nstorj.io/storj/private/server.(*tcpUserTimeoutListener).Accept(0xc000396058?)\n\t/go/src/storj.io/storj/private/server/listener.go:46 +0x25\nstorj.io/drpc/drpcmigrate.(*ListenMux).monitorBase(0xc00022bb20)\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmigrate/mux.go:115 +0x3a\ncreated by storj.io/drpc/drpcmigrate.(*ListenMux).Run in goroutine 1107\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmigrate/mux.go:88 +0x11b\n\ngoroutine 9 [syscall, 1246 minutes]:\nos/signal.signal_recv()\n\t/usr/local/go/src/runtime/sigqueue.go:152 +0x29\nos/signal.loop()\n\t/usr/local/go/src/os/signal/signal_unix.go:23 +0x13\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/local/go/src/os/signal/signal.go:151 +0x1f\n\ngoroutine 10 [chan receive, 1246 minutes]:\nstorj.io/common/debug.(*Server).Run.func3()\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/debug/server.go:182 +0x35\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78 +0x56\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 7\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:75 +0x96\n\ngoroutine 11 [chan receive, 1246 minutes]:\nstorj.io/drpc/drpcmigrate.(*ListenMux).Run(0xc00022a2a0, {0x157f338?, 0xc00022e7d0?})\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmigrate/mux.go:90 +0x12e\nstorj.io/common/debug.(*Server).Run.func4()\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/debug/server.go:186 +0x31\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78 +0x56\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 7\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:75 +0x96\n\ngoroutine 12 [select, 1246 minutes]:\nstorj.io/drpc/drpcmigrate.(*listener).Accept(0xc00021c7c0)\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmigrate/listener.go:37 +0x7e\nnet/http.(*Server).Serve(0xc000002040, {0x15798a0, 0xc00021c7c0})\n\t/usr/local/go/src/net/http/server.go:3056 +0x364\nstorj.io/common/debug.(*Server).Run.func5()\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/debug/server.go:195 +0x53\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78 +0x56\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 7\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:75 +0x96\n\ngoroutine 21
(...)
go/pkg/mod/storj.io/drpc@v0.0.34/drpcserver/server.go:109 +0x1e5\nstorj.io/drpc/drpcserver.(*Server).Serve.func2({0x157fa18, 0xc00036aea0})\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcserver/server.go:157 +0x59\nstorj.io/drpc/drpcctx.(*Tracker).track(0xc00036aea0, 0xc09e2f8800?)\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcctx/tracker.go:35 +0x2e\ncreated by storj.io/drpc/drpcctx.(*Tracker).Run in goroutine 1111\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcctx/tracker.go:30 +0x79\n\ngoroutine 1243081 [sync.Cond.Wait, 684 minutes]:\nsync.runtime_notifyListWait(0xc01c448bf8, 0x0)\n\t/usr/local/go/src/runtime/sema.go:527 +0x159\nsync.(*Cond).Wait(0xc00564af10?)\n\t/usr/local/go/src/sync/cond.go:70 +0x85\nstorj.io/common/sync2.(*Throttle).ConsumeOrWait(0xc01c448be0, 0x4000)\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/sync2/throttle.go:50 +0x8f\nstorj.io/storj/storagenode/piecestore.(*Endpoint).Download.func7()\n\t/go/src/storj.io/storj/storagenode/piecestore/endpoint.go:760 +0x165\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78 +0x56\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1142886\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:75 +0x96\n\ngoroutine 1220625 [select, 634 minutes]:\nstorj.io/drpc/drpcmanager.(*Manager).manageReader(0xc04add2d20)\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmanager/manager.go:265 +0x325\ncreated by storj.io/drpc/drpcmanager.NewWithOptions in goroutine 1220624\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmanager/manager.go:121 +0x416\n\ngoroutine 1220624 [semacquire, 632 minutes]:\nsync.runtime_Semacquire(0xc156225380?)\n\t/usr/local/go/src/runtime/sema.go:62 +0x25\nsync.(*WaitGroup).Wait(0x0?)\n\t/usr/local/go/src/sync/waitgroup.go:116 +0x48\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc02e00a580)\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:56 +0x25\nstorj.io/storj/storagenode/piecestore.(*Endpoint).Download(0xc0005461e0, {0x1582fc0, 0xc053f46310})\n\t/go/src/storj.io/storj/storagenode/piecestore/endpoint.go:842 +0x20f0\nstorj.io/common/pb.DRPCPiecestoreDescription.Method.func2({0x11afba0?, 0xc0005461e0}, {0xc156561680?, 0x1f?}, {0x10faec0?, 0xc00a5dcbe0}, {0x156d378?, 0x0?})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/pb/piecestore2_drpc.pb.go:302 +0xab\nstorj.io/drpc/drpcmux.(*Mux).HandleRPC(0xc00097ee00?, {0x1580040, 0xc00a5dcbe0}, {0xc156561680, 0x1f})\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcmux/handle_rpc.go:33 +0x20d\nstorj.io/common/rpc/rpctracing.(*Handler).HandleRPC(0xc00054c348, {0x15803c0, 0xc00a5dcba0}, {0xc156561680, 0x1f})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/rpc/rpctracing/handler.go:61 +0x2e3\nstorj.io/common/experiment.(*Handler).HandleRPC(0xc00040c390, {0x1580500, 0xc0f077e240}, {0xc156561680, 0x1f})\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/experiment/import.go:42 +0x167\nstorj.io/drpc/drpcserver.(*Server).handleRPC(0xc04add2d20?, 0x157f300?, {0xc156561680?, 0x1db5f80?})\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcserver/server.go:167 +0x42\nstorj.io/drpc/drpcserver.(*Server).ServeOne(0xc0004121e0, {0x157fa18, 0xc00036af60}, {0x1579180?, 0xc0105bb440?})\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcserver/server.go:109 +0x1e5\nstorj.io/drpc/drpcserver.(*Server).Serve.func2({0x157fa18, 0xc00036af60})\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcserver/server.go:157 +0x59\nstorj.io/drpc/drpcctx.(*Tracker).track(0xc00036af60, 0xc16229a490?)\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcctx/tracker.go:35 +0x2e\ncreated by storj.io/drpc/drpcctx.(*Tracker).Run in goroutine 1113\n\t/go/pkg/mod/storj.io/drpc@v0.0.34/drpcctx/tracker.go:30 +0x79\n\ngoroutine 1313046 [sync.Cond.Wait, 634 minutes]:\nsync.runtime_notifyListWait(0xc01cc5cc98, 0x0)\n\t/usr/local/go/src/runtime/sema.go:527 +0x159\nsync.(*Cond).Wait(0xc2be694f10?)\n\t/usr/local/go/src/sync/cond.go:70 +0x85\nstorj.io/common/sync2.(*Throttle).ConsumeOrWait(0xc01cc5cc80, 0x4000)\n\t/go/pkg/mod/storj.io/common@v0.0.0-20240425113201-9815a85cbc32/sync2/throttle.go:50 +0x8f\nstorj.io/storj/storagenode/piecestore.(*Endpoint).Download.func7()\n\t/go/src/storj.io/storj/storagenode/piecestore/endpoint.go:760 +0x165\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78 +0x56\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1220624\n\t/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:75 +0x96\n\ngoroutine 2045207 [select, 16 minutes]:\ndatabase/sql.(*DB).connectionCleaner(0xc0004181a0, 0x0?)\n\t/usr/local/go/src/database/sql/sql.go:1061 +0x9c\ncreated by database/sql.(*DB).startCleanerLocked in goroutine 2045161\n\t/usr/local/go/src/database/sql/sql.go:1048 +0x105\n\ngoroutine 2550881 [select, 3 minutes]:\ndatabase/sql.(*DB).connectionCleaner(0xc0002b7110, 0x1d0f430?)\n\t/usr/local/go/src/database/sql/sql.go:1061 +0x9c\ncreated by database/sql.(*DB).startCleanerLocked in goroutine 2550619\n\t/usr/local/go/src/database/sql/sql.go:1048 +0x105\n"}
Could someone elaborate on this?
And moreover, can we get readable info instead of these insane errors?
Although no real errors, the node only shows version checks after this ‘INFO’ line. Which is now dating two hours back.
I had to cut out a part from the error due to length restrictions (32000; error was over 130000 chars).