暗能星系

zhanglu

[root@node3 cephfs_data]# ps -eo pid,stat,cmd | grep -E '^.+ D'
647454 D sync
773495 D sync
777729 D /usr/bin/mount -o bind,remount /proc/7817/fd/50 /var/lib/kubelet/pods/ce2112de-6092-400b-8d1f-ff9c7f0feb60/volume-subpaths/data/cromwell/0
807143 D sync
831541 D sync
862778 D sync
923295 D sync
925182 D sync
1120488 D sync
1137925 D sync
1438346 D sync
1685722 D /usr/bin/mount -o bind,remount /proc/7817/fd/54 /var/lib/kubelet/pods/01c428af-c57a-4159-93ce-4e57c73ed85b/volume-subpaths/data/cromwell/0
1698130 D sync
1699192 D sync
1729571 D sync
1729918 D sync
1916667 D sync
1957175 D sync
3327874 S+ grep --color=auto -E ^.+ D
3402901 DNs (coredump)
3883466 D+ rm -rf ./5b9c9af2-49ab-4cd3-bbe9-637c131ef344 ./ff853c49-720c-442a-b5bf-fb50c63736c2 ./4dba7671-4615-4ce8-8083-09c1b6a3b895 ./a00fd097-cd21-4561-9dcc-2bb9b66d58fa ./2ca5d416-7398-43f4-b368-68e790abaf95 ./f7f4938d-0b2a-4e95-9f6c-0b9e3c442ef6 ./5958823a-cb9b-4556-a30d-4142eba696c7 ./8281f2d8-fcb9-4508-853d-be96fe10535e ./0ee9f259-014c-44e1-9453-f2af6f423c06 ./4fbe43e5-4018-434f-a6fd-61ba3ebc69e6 ./03a22cb6-5ca6-4b58-8f32-b95988a4605c ./58ade6fd-845a-4ee5-bebd-b0dce6cfdac5 ./4b8fde55-6387-4fd3-9092-d4c19c44ab9c ./aed461b9-dfff-4fc1-b089-f33b2673b1c5 ./7a932a56-0a47-42d4-893b-d0435a2edc1c ./57ff22a3-5fa4-4d77-94b7-71011cb6238e ./e8c72d7c-1d92-4d3c-8702-f4cf50ea2ee2 ./93a9de1b-7be3-40ba-9e9e-3364b67b202d ./e3d77188-3693-4f7f-81e7-8688b4b01dec ./8bf3777b-361c-46c1-9328-c9f3d5cc2822 ./eaf418dc-01b3-4bc0-b272-d2ba3f2493ce ./1c24a9f4-2e97-44c8-af0a-85db7e374956 ./77b44fc3-2f4b-480b-b01a-d57a1395fe1e ./f2fd7fe2-3903-437b-a40f-2e812dc35201 ./a862e7a9-99e2-4c31-82ba-3e87c0421354 ./b16de460-9245-41d5-8b69-444e95e549a0 ./815801ee-1ded-46db-933a-0decd35d5fe4 ./75f6352d-2080-40dc-9ac5-ce4c6566e0c2 ./9ef6a0eb-1d37-4966-b7e1-1ffc4b9abc22 ./d97f36bd-6805-4bfa-a386-0b90bb42d8cd ./345b403e-45e2-4e09-9b99-23b1b3c7f33c ./c5ef2a0e-2868-4e09-9c26-cf5357f60448 ./b0b9b6c3-bcb6-479f-83c5-429e306ff1ad ./4ec53f20-4f2c-430c-a332-7736584b26c7 ./02ac4847-d640-4312-a761-e7fd01d45c27 ./53630622-fa7f-4cf6-91b2-12768883af6f ./e1660f00-841f-4899-a4b2-c2451edf2b36 ./07de3f21-1de8-4326-9bf7-5774a2791eaa ./97914a70-2fe4-4c1f-a326-adebe8222df0 ./695f925d-723b-4446-b9b1-11dfa642f065
3911819 D sync
3948171 D sync

zhanglu

SELECT "METADATA_KEY", COUNT()
FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = 'e886888f-c972-4932-a9a4-187525877ea0'
GROUP BY "METADATA_KEY"
ORDER BY COUNT() DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = '32d44941-c654-4163-b01e-aa41a0cc597a'
AND "METADATA_KEY" = 'vmStartTime'

          METADATA_KEY              |  count

zhanglu

e886888f-c972-4932-a9a4-187525877ea0
b9eec03b-590a-4a53-8a99-088fe01c9070
2e97dac5-f31b-4cbe-8712-4610295c6dfa
731bad53-fe54-43a2-b1d1-2639ac371a43

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b9eec03b-590a-4a53-8a99-088fe01c9070' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2e97dac5-f31b-4cbe-8712-4610295c6dfa' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '731bad53-fe54-43a2-b1d1-2639ac371a43' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" in (SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;);

zhanglu

zhanglu

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '9992a98d-9fbc-48a5-bf0f-13af49246128' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'ec7b2a76-ae8a-45f9-b484-5346b6f74efd' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f44f2c67-22bd-4953-92d9-8b2da5207a55' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '346c5771-8908-43b9-a037-416d98f0f8f8' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'af81e1d7-c5e9-4852-b60d-a66196d37705' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f3aaa31b-59f6-4e35-930d-5fdc96e69960' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a9fb5224-ede5-4227-a755-dae75c5e0ffc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e1d59bbb-a51e-471c-9efb-50d4dbb5affc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3ed05f3a-1370-457a-9af7-8f70963afe6d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3192edd4-c3cd-4b56-84c4-d78c80b64287' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a61c1d9b-86b7-4455-85a7-e5435ac57ab5' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2ee83a60-3d6e-4ea8-8616-18bff20e1cb0' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b853a6ea-f7a8-4b14-987b-7152373151ec' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '23fbc5cb-be12-4e80-8510-ab1a10774675' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '8ebdadd1-a61d-419e-9ce0-80b8cac41e9c' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6da5292a-7d2f-40a8-aea3-d4e1b50052b6' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '20283bc4-aacd-47bb-8c73-f450ea432bd7' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e2ca06ea-5df1-4e0b-83e3-2f0574632829' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e6f72645-4253-4581-85bf-e4cda8f51a13' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

670765 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/metage_megahit/9992a98d-9fbc-48a5-bf0f-13af49246128/
832494 D /usr/bin/mount -o bind,remount /proc/6616/fd/62 /var/lib/kubelet/pods/61ee2602-a811-4953-8fbe-c4f1afca762a/volume-subpaths/data/cromwell-runner/0
833804 D /usr/bin/mount -o bind,remount /proc/6616/fd/49 /var/lib/kubelet/pods/2093bbf3-eb23-4139-8076-5bca5f720d1b/volume-subpaths/data/cromwell-runner/0
989841 D /usr/bin/mount -o bind,remount /proc/6616/fd/70 /var/lib/kubelet/pods/322e9748-cb86-4355-9d7d-0e106533cf0a/volume-subpaths/data/cromwell-runner/0
1360682 D sync
1476802 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2303814 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2336314 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2668230 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/rnaseq_eukaryonanalysis/e2ca06ea-5df1-4e0b-83e3-2f0574632829/

zhanglu

strace sync
execve("/usr/bin/sync", ["sync"], 0x7ffe4aa407b0 /* 33 vars /) = 0
brk(NULL) = 0x55f54a1c7000
arch_prctl(0x3001 / ARCH_??? */, 0x7fff30c2dda0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56063, ...}) = 0
mmap(NULL, 56063, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f867bce6000
close(3) = 0
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2405\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=3167872, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f867bce4000
lseek(3, 808, SEEK_SET) = 808
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
mmap(NULL, 3950400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f867b703000
mprotect(0x7f867b8bf000, 2093056, PROT_NONE) = 0
mmap(0x7f867babe000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f867babe000
mmap(0x7f867bac4000, 14144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f867bac4000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7f867bce5540) = 0
mprotect(0x7f867babe000, 16384, PROT_READ) = 0
mprotect(0x55f549b32000, 4096, PROT_READ) = 0
mprotect(0x7f867bcf4000, 4096, PROT_READ) = 0
munmap(0x7f867bce6000, 56063) = 0
brk(NULL) = 0x55f54a1c7000
brk(0x55f54a1e8000) = 0x55f54a1e8000
brk(NULL) = 0x55f54a1e8000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2997, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2997
read(3, "", 4096) = 0
close(3) = 0
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
sync(

zhanglu

ps -eo pid,ppid,state,cmd | grep ^D

zhanglu

timedatectl set-timezone Asia/Shanghai

zhanglu

docker pull dockerhub.genostack.com:8090/library/genostack_kubernetes_terminal:v0.8

docker pull dockerhub.genostack.com:8090/angs/analysis_chemical_shift_table:V03.02

docker pull dockerhub.genostack.com:8090/angs/nmrium:v01

docker pull dockerhub.genostack.com:8090/library/sanshu_database:latest

docker pull dockerhub.genostack.com:8090/library/sanshu_mutant_database:V1

zhanglu

docker save -o prom_alertmanager_v0.23.0.tar prom/alertmanager:v0.23.0

docker save -o kubesphere_prometheus-config-reloader_v0.55.1.tar kubesphere/prometheus-config-reloader:v0.55.1

docker save -o kubesphere_kube-state-metrics_v2.6.0.tar kubesphere/kube-state-metrics:v2.6.0

docker save -o kubesphere_kube-rbac-proxy_v0.11.0.tar kubesphere/kube-rbac-proxy:v0.11.0

docker save -o dockerhub.genostack.com_grafana_12.0.2-security-01-ubuntu.tar dockerhub.genostack.com:8090/library/grafana:12.0.2-security-01-ubuntu

docker save -o prom_node-exporter_v1.3.1.tar prom/node-exporter:v1.3.1

docker save -o kubesphere_notification-manager_v2.3.0.tar kubesphere/notification-manager:v2.3.0

docker save -o kubesphere_notification-tenant-sidecar_v3.2.0.tar kubesphere/notification-tenant-sidecar:v3.2.0

docker save -o kubesphere_notification-manager-operator_v2.3.0.tar kubesphere/notification-manager-operator:v2.3.0

docker save -o prom_prometheus_v2.39.1.tar prom/prometheus:v2.39.1

docker save -o kubesphere_prometheus-operator_v0.55.1.tar kubesphere/prometheus-operator:v0.55.1

docker save -o thanosio_thanos_v0.31.0.tar thanosio/thanos:v0.31.0

zhanglu

c-48ed-b0fe-6af0e10bfe1a")
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.719980 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data podName:404b6cca-8dec-48ed-b0fe-6af0e10bfe1a nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.71995658 +0800 CST m=+85903.723576802 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data") pod "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a" (UID: "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a") : error processing /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.720008 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data podName:599e5d21-3c5f-4898-a87a-38bf7e15e8f3 nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.719990596 +0800 CST m=+85903.723610810 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data") pod "599e5d21-3c5f-4898-a87a-38bf7e15e8f3" (UID: "599e5d21-3c5f-4898-a87a-38bf7e15e8f3") : error processing /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"

zhanglu

复制日志里的挂载路径，查询占用进程

lsof | grep "/var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0"
lsof | grep "/var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0"

zhanglu

Check for open files in that subpath

sudo lsof /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

OR use fuser to see PIDs

sudo fuser -v /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

sudo umount -l /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

root 92467 1 0 11:22 ? 00:00:00 /usr/bin/mount -o bind,remount /proc/3202/fd/48 /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

dmesg -T | grep -i "blocked for more than 120 seconds"

zhanglu

Run ceph -s to check general health.

Run ceph tell mds.* dump captures to see if there are ongoing slow requests or if an MDS is "laggy."

zhanglu

[root@node2 0]# mount | grep ceph
tmpfs on /var/lib/kubelet/pods/9d7dc27a-6ee9-4c85-bc86-a9df68a7b559/volumes/kubernetes.io~secret/rook-csi-cephfs-plugin-sa-token-pb2sn type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/f8dd6c09-38de-4009-a3fb-8c4b5f35e009/volumes/kubernetes.io~projected/ceph-csi-configs type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/371a1859-be44-4db2-b345-642b8c27f6de/volumes/kubernetes.io~secret/rook-ceph-mds-myfs-ec-a-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/e4938d1d-37bb-45cd-97a3-ae836c7f9e25/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/326f83ec-27bb-4b20-8ef5-df9f575f0b91/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ae25aa0a-fbf3-4786-b457-cdaddd158305/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/e4938d1d-37bb-45cd-97a3-ae836c7f9e25/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/37b1b062-36dc-4653-94bc-d65eb9471be5/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ee81bbf3-7678-40e4-8bc7-a273ed4161db/volumes/kubernetes.io~secret/rook-ceph-crash-collector-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/2cf5bdfa-1592-4e7a-89ec-4282784d8c8d/volumes/kubernetes.io~secret/rook-ceph-mons-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/bcd70180-7a0b-45bc-a730-3370b421cda2/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/37b1b062-36dc-4653-94bc-d65eb9471be5/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/01448345-5e05-400a-a5bc-00665b06534d/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ae25aa0a-fbf3-4786-b457-cdaddd158305/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/326f83ec-27bb-4b20-8ef5-df9f575f0b91/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/bcd70180-7a0b-45bc-a730-3370b421cda2/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/af822db5-cdc0-4d81-9ec9-6a9a00229d19/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/01448345-5e05-400a-a5bc-00665b06534d/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/af822db5-cdc0-4d81-9ec9-6a9a00229d19/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/volumes/csi/csi-vol-42070b59-05af-11ee-a677-b62e185b651a/8d7c3218-4807-499d-bc93-bfff24ceb882 on /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-23b68c83-5c16-44a2-8d78-71ea15a1dff3/globalmount type ceph (rw,relatime,name=csi-cephfs-node,secret=<hidden>,fsid=00000000-0000-0000-0000-000000000000,acl,mds_namespace=myfs-ec,_netdev)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/volumes/csi/csi-vol-42070b59-05af-11ee-a677-b62e185b651a/8d7c3218-4807-499d-bc93-bfff24ceb882 on /var/lib/kubelet/pods/12afe02d-0149-491d-8707-ff8ea97af49f/volumes/kubernetes.io~csi/pvc-23b68c83-5c16-44a2-8d78-71ea15a1dff3/mount type ceph (rw,relatime,name=csi-cephfs-node,secret=<hidden>,fsid=00000000-0000-0000-0000-000000000000,acl,mds_namespace=myfs-ec,_netdev)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /cephfs_data type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/ee4a50ef-e31b-439b-b1e1-200aa6527c0a/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/ee4a50ef-e31b-439b-b1e1-200aa6527c0a/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0fb12d38-0ea7-4eab-81c1-44003a0ee4a2/volume-subpaths/data/cromwell-summarizer/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0fb12d38-0ea7-4eab-81c1-44003a0ee4a2/volume-subpaths/data/cromwell-summarizer/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/515a95d8-db33-48b3-82a9-2a3a24a69d5f/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/515a95d8-db33-48b3-82a9-2a3a24a69d5f/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/111df690-fc38-4db1-a2e5-70447f9e80ae/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/111df690-fc38-4db1-a2e5-70447f9e80ae/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/e439a245-ea34-4797-b4b9-3517f9d62650/volume-subpaths/data/websocket/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/a127b7f9-38df-457d-9227-9bab7725f86c/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/a127b7f9-38df-457d-9227-9bab7725f86c/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3b020b6c-7511-486a-8c05-19d719989515/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3b020b6c-7511-486a-8c05-19d719989515/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a6e427f-03f7-4496-b66e-9ad23cb80d67/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a6e427f-03f7-4496-b66e-9ad23cb80d67/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3926f598-a2a7-4f09-a6a7-64bebb7bb96d/volume-subpaths/data/phpfpm/6 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/cc6add18-2573-4a3f-9635-3e6a0cb18508/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/379b60cd-4173-4db2-ae42-6df3f6ff4b5a/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/97aff715-f0b3-4bd9-b6ef-7630cd9a7cdd/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/017ecee2-c97b-4e13-9529-1a7d39e449e9/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/14656dfa-19fb-4ade-8472-7ded179ccec9/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/9f5ea36b-8596-4ae8-a143-0466fa418705/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a328296-e0f5-4e7c-8def-d571becb3f9b/volume-subpaths/data/ms-hippo-flow/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/c4b9e777-73b0-4f48-b73b-6f851e1a133a/volume-subpaths/data/ms-hippo-flow/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0e905f16-fb54-4960-a814-8788a7fa1a5e/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/aa0880e7-5b3c-4608-9769-3bc08473f00f/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/9e5545a5-5a4f-4e1e-a2e0-fc17a16783e5/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/067e33e7-1364-48d8-ab7e-1045d791170d/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)

zhanglu

ceph tell mds.myfs-ec session ls
ceph tell mds.myfs-ec client evict id=<client-id>