Closed
Description
with insufficient memory, all different weird kind of issues happen, seen several tests frequently failing b/c of that, and containerd timeouts are one of them
example:
https://storage.googleapis.com/minikube-builds/logs/11006/74537ca/KVM_Linux_containerd.html#fail_TestForceSystemdFlag
https://storage.googleapis.com/minikube-builds/logs/11006/74537ca/KVM_Linux_containerd.html#fail_TestForceSystemdEnv
❯ minikube start -p memless --memory=1800 --force-systemd --alsologtostderr -v=9 --driver=kvm2 --container-runtime=containerd
...
I0408 23:23:42.109960 32694 crio.go:119] couldn't verify netfilter by "sudo sysctl net.bridge.bridge-nf-call-iptables" which might be okay. error: sudo sysctl net.bridge.bridge-nf-call-iptables: Process exited with status 255
...
W0408 23:25:18.298509 32694 out.go:222] ❌ Exiting due to RUNTIME_ENABLE: stat /run/containerd/containerd.sock: Process exited with status 1
stdout:
stderr:
stat: cannot stat '/run/containerd/containerd.sock': No such file or directory
...
inside the vm:
# crictl images --output json
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0x11121fe, 0x16)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/panic.go:1116 +0x72 fp=0x7fff8c7be3d0 sp=0x7fff8c7be3a0 pc=0x435bb2
runtime.sysMap(0xc000000000, 0x4000000, 0x1917418)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mem_linux.go:169 +0xc6 fp=0x7fff8c7be410 sp=0x7fff8c7be3d0 pc=0x418f86
runtime.(*mheap).sysAlloc(0x18fc580, 0x400000, 0x0, 0x4)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/malloc.go:727 +0x1e5 fp=0x7fff8c7be4b8 sp=0x7fff8c7be410 pc=0x40c925
runtime.(*mheap).grow(0x18fc580, 0x1, 0x0)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mheap.go:1344 +0x85 fp=0x7fff8c7be520 sp=0x7fff8c7be4b8 pc=0x4284a5
runtime.(*mheap).allocSpan(0x18fc580, 0x1, 0x656c6d656d002a00, 0x1917428, 0x0)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mheap.go:1160 +0x6b6 fp=0x7fff8c7be5a0 sp=0x7fff8c7be520 pc=0x428256
runtime.(*mheap).alloc.func1()
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mheap.go:907 +0x65 fp=0x7fff8c7be5f8 sp=0x7fff8c7be5a0 pc=0x461b65
runtime.(*mheap).alloc(0x18fc580, 0x1, 0x12a, 0x0)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mheap.go:901 +0x85 fp=0x7fff8c7be648 sp=0x7fff8c7be5f8 pc=0x427725
runtime.(*mcentral).grow(0x190f438, 0x0)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mcentral.go:506 +0x7a fp=0x7fff8c7be690 sp=0x7fff8c7be648 pc=0x41895a
runtime.(*mcentral).cacheSpan(0x190f438, 0x40000)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mcentral.go:177 +0x3e5 fp=0x7fff8c7be708 sp=0x7fff8c7be690 pc=0x4186e5
runtime.(*mcache).refill(0x7f501fdc6108, 0x2a)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/mcache.go:142 +0xa5 fp=0x7fff8c7be728 sp=0x7fff8c7be708 pc=0x418085
runtime.(*mcache).nextFree(0x7f501fdc6108, 0x18e482a, 0x7f501fdc6108, 0x7f4ffb4a6000, 0x7fff8c7be7b8)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/malloc.go:880 +0x8d fp=0x7fff8c7be760 sp=0x7fff8c7be728 pc=0x40d1ad
runtime.mallocgc(0x180, 0x10fbec0, 0x7fff8c7be801, 0x7fff8c7be860)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/malloc.go:1061 +0x834 fp=0x7fff8c7be800 sp=0x7fff8c7be760 pc=0x40db94
runtime.newobject(0x10fbec0, 0x4608a0)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/malloc.go:1195 +0x38 fp=0x7fff8c7be830 sp=0x7fff8c7be800 pc=0x40e038
runtime.malg(0x8000, 0x0)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/proc.go:3493 +0x31 fp=0x7fff8c7be870 sp=0x7fff8c7be830 pc=0x440751
runtime.mpreinit(0x18e4820)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/os_linux.go:340 +0x29 fp=0x7fff8c7be890 sp=0x7fff8c7be870 pc=0x432969
runtime.mcommoninit(0x18e4820, 0xffffffffffffffff)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/proc.go:663 +0xf7 fp=0x7fff8c7be8d8 sp=0x7fff8c7be890 pc=0x4399b7
runtime.schedinit()
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/proc.go:565 +0xa5 fp=0x7fff8c7be930 sp=0x7fff8c7be8d8 pc=0x439545
runtime.rt0_go(0x7fff8c7be968, 0x4, 0x7fff8c7be968, 0x0, 0x0, 0x4, 0x7fff8c7beefe, 0x7fff8c7bef05, 0x7fff8c7bef0c, 0x7fff8c7bef15, ...)
/nix/store/r5wsnvxfc14badlg8fcx4fyvji3lksz6-go-1.15/share/go/src/runtime/asm_amd64.s:214 +0x125 fp=0x7fff8c7be938 sp=0x7fff8c7be930 pc=0x468fc5
# free
total used free shared buff/cache available
Mem: 1782984 129504 80684 1551104 1572796 17924
Swap: 0 0 0
/assign