How to kill a docker swarm

Assign way more replicas than you have of memory on all nodes and watch the Swarm crash which can easily reproduce in a small VM for testing.

root@Deb11Docker01:~# docker ps
CONTAINER ID   IMAGE     COMMAND                  CREATED        STATUS        PORTS      NAMES
1336c9b0275e   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.7.ue8ff3g3ef2pv4ezu4cy5qc23
f5496728a590   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.11.kk3ij52i7dq0p4xqe3r05apzc
b0b2a47dc6b6   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.8.p5woskaqpyc6kod0njfmdq5m9
6dc2c2a0f8f2   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.23.x27zfpun3mf8j40qofers8mqs
91e264dc4ec9   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.17.lma6im540lw77uk6nolxg7rej
47830106e26d   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.18.8y6rm833sgxvbtnj1np4e8q0i
a4246ca036e9   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.19.z7jj4hiz6sjyux2hmgy6d9449
965754f35798   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.13.ue6b58bkx4w44n8gkht0bu0r8
d4a9572cae71   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.6.4sa0179qxpoy4m522d5mc2shy
1ea0cd1e493c   redis:5   "docker-entrypoint.s…"   13 hours ago   Up 13 hours   6379/tcp   clever_bouman.28.nmzn1jd77lybe211thm1f9q8n
root@Deb11Docker01:~# docker service update --replicas=60 clever_bouman
clever_bouman
overall progress: 51 out of 60 tasks
Error response from daemon: rpc error: code = DeadlineExceeded desc = context deadline exceeded
root@Deb11Docker01:~# docker service update --replicas=40 clever_bouman
Error response from daemon: rpc error: code = Unknown desc = The swarm does not have a leader. It's possible that too few managers are online. Make sure more than half of the managers are online.

[137927.550974] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=docker.service,mems_allowed=0,global_oom,task_memcg=/system.slice/docker.service,task=dockerd,pid=55994,uid=0
[137927.551065] Out of memory: Killed process 55994 (dockerd) total-vm:1859264kB, anon-rss:80588kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:664kB oom_score_adj:0



root@Deb11Docker02:~# ps aux|grep container
root        1404  0.0  3.5 1370844 35244 ?       Ssl  Nov30   1:44 /usr/bin/containerd
root       48601  1.3  0.3 1328876 3528 ?        Sl   02:34   0:08 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 8be1aac07725843e36d47934209ee280ec2705df3bd098a9320eb0abd659eaaa -address /run/containerd/containerd.sock
root       49954  0.9  0.4 1329132 4456 ?        Sl   02:35   0:05 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 3445ae9397be1d9cbb80dcf9679c7765c2e6f5093672543b4c16d5b0e7935c71 -address /run/containerd/containerd.sock
root       50004  0.9  1.4 1322088 14560 ?       Sl   02:35   0:05 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 72301e658829c8fc3679b0744a110100cd31423c6f510bf330e8b90842c308cb -address /run/containerd/containerd.sock
root       50037  0.7  0.6 1314148 6360 ?        Sl   02:35   0:04 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 13f22679e833befe833f14073d800d6fa6a04e44bb30e317dc01a282b19a3e1b -address /run/containerd/containerd.sock
root       50045  0.7  0.4 1322088 4560 ?        Sl   02:35   0:04 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 1c0e7f3e65032590ac3078afc2b227c1003680c7b8c48c755193a27b17027aec -address /run/containerd/containerd.sock
root       50061  0.7  0.6 1328940 6472 ?        Sl   02:35   0:04 /usr/bin/containerd-shim-runc-v2 -namespace moby -id b258bac1daebc0eebc75195802f22631991b99effda0c8dd32d916dda53b3ecd -address /run/containerd/containerd.sock
root       50109  0.7  0.4 1322088 4672 ?        Sl   02:35   0:04 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 5d9b13fc6a53e71a9b0efcf9524b4b410843a6351b273020cb38bd49fdeda8d1 -address /run/containerd/containerd.sock
root       50150  1.0  0.6 1320680 6564 ?        Sl   02:35   0:05 /usr/bin/containerd-shim-runc-v2 -namespace moby -id cb18a243b3cc30d4d62d9fec8995c14da81e8c2a3ed6de670a880a4e174c4e55 -address /run/containerd/containerd.sock
root       51985  0.0  0.0   6180   716 pts/0    S+   02:44   0:00 grep container
root@Deb11Docker02:~# killall -9 containerd-shim-runc-v2
root@Deb11Docker02:~# ps aux|grep container
root        1404  0.0  3.5 1379040 35244 ?       Ssl  Nov30   1:44 /usr/bin/containerd
root       52092  0.0  0.0   6180   716 pts/0    S+   02:45   0:00 grep container
root@Deb11Docker02:~#


 docker swarm init --force-new-cluster


Tags:

docker, swarmassign, replicas, nodes, swarm, reproduce, vm, deb, ps, container, ports, redis, quot, entrypoint, hellip, tcp, clever_bouman, ue, ff, ef, pv, ezu, cy, qc, kk, ij, dq, xqe, apzc, dc, woskaqpyc, kod, njfmdq, zfpun, mf, qofers, mqs, ec, lma, im, lw, uk, nolxg, rej, rm, sgxvbtnj, np, ca, z, jj, hiz, sjyux, hmgy, bkx, gkht, bu, cae, sa, qxpoy, mc, ea, nmzn, jd, lybe, thm, update, overall, tasks, daemon, rpc, deadlineexceeded, desc, context, deadline, exceeded, managers, online, oom, constraint, constraint_none, nodemask, null, cpuset, mems_allowed, global_oom, task_memcg, slice, task, dockerd, pid, uid, kb, anon, rss, shmem, pgtables, oom_score_adj, aux, grep, ssl, nov, usr, bin, containerd, sl, shim, runc, namespace, moby, aac, ee, df, bd, eb, abd, eaaa, ae, cbb, dcf, fc, bf, cb, befe, fa, bb, ac, afc, aec, bac, daebc, eebc, effda, dd, dda, ecd, efcf, fdeda, cc, fec, pts, killall, init, cluster,

Latest Articles

  • FreePBX 17 How To Add a Trunk
  • Docker Container Onboot Policy - How to make sure a container is always running
  • FreePBX 17 How To Add Phones / Extensions and Register
  • Warning: The driver descriptor says the physical block size is 2048 bytes, but Linux says it is 512 bytes. solution
  • Cisco How To Use a Third Party SIP Phone (eg. Avaya, 3CX)
  • Cisco Unified Communication Manager (CUCM) - How To Add Phones
  • pptp / pptpd not working in DD-WRT iptables / router
  • systemd-journald high memory usage solution
  • How to Install FreePBX 17 in Linux Debian Ubuntu Mint Guide
  • How To Install Cisco's CUCM (Cisco Unified Communication Manager) 12 Guide
  • Linux Ubuntu Redhat How To Extract Images from PDF
  • Linux and Windows Dual Boot Issue NIC Won't work After Booting Windows
  • Cisco CME How To Enable ACD hunt groups
  • How to install gns3 on Linux Ubuntu Mint
  • How to convert audio for Asterisk .wav format
  • Using Cisco CME Router with Asterisk as a dial-peer
  • Cisco CME How To Configure SIP Trunk VOIP
  • Virtualbox host Only Network Error Failed to save host network interface parameter - Cannot change gateway IP of host only network
  • Cisco CME and C7200 Router Testing and Learning Environment on Ubuntu 20+ Setup Tutorial Guide
  • Abusive IP ranges blacklist