暗能星系

zhanglu

SELECT "METADATA_KEY", COUNT()
FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = 'e886888f-c972-4932-a9a4-187525877ea0'
GROUP BY "METADATA_KEY"
ORDER BY COUNT() DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = '32d44941-c654-4163-b01e-aa41a0cc597a'
AND "METADATA_KEY" = 'vmStartTime'

          METADATA_KEY              |  count

zhanglu

e886888f-c972-4932-a9a4-187525877ea0
b9eec03b-590a-4a53-8a99-088fe01c9070
2e97dac5-f31b-4cbe-8712-4610295c6dfa
731bad53-fe54-43a2-b1d1-2639ac371a43

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b9eec03b-590a-4a53-8a99-088fe01c9070' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2e97dac5-f31b-4cbe-8712-4610295c6dfa' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '731bad53-fe54-43a2-b1d1-2639ac371a43' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" in (SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;);

zhanglu

zhanglu

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '9992a98d-9fbc-48a5-bf0f-13af49246128' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'ec7b2a76-ae8a-45f9-b484-5346b6f74efd' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f44f2c67-22bd-4953-92d9-8b2da5207a55' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '346c5771-8908-43b9-a037-416d98f0f8f8' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'af81e1d7-c5e9-4852-b60d-a66196d37705' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f3aaa31b-59f6-4e35-930d-5fdc96e69960' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a9fb5224-ede5-4227-a755-dae75c5e0ffc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e1d59bbb-a51e-471c-9efb-50d4dbb5affc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3ed05f3a-1370-457a-9af7-8f70963afe6d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3192edd4-c3cd-4b56-84c4-d78c80b64287' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a61c1d9b-86b7-4455-85a7-e5435ac57ab5' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2ee83a60-3d6e-4ea8-8616-18bff20e1cb0' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b853a6ea-f7a8-4b14-987b-7152373151ec' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '23fbc5cb-be12-4e80-8510-ab1a10774675' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '8ebdadd1-a61d-419e-9ce0-80b8cac41e9c' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6da5292a-7d2f-40a8-aea3-d4e1b50052b6' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '20283bc4-aacd-47bb-8c73-f450ea432bd7' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e2ca06ea-5df1-4e0b-83e3-2f0574632829' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e6f72645-4253-4581-85bf-e4cda8f51a13' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

670765 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/metage_megahit/9992a98d-9fbc-48a5-bf0f-13af49246128/
832494 D /usr/bin/mount -o bind,remount /proc/6616/fd/62 /var/lib/kubelet/pods/61ee2602-a811-4953-8fbe-c4f1afca762a/volume-subpaths/data/cromwell-runner/0
833804 D /usr/bin/mount -o bind,remount /proc/6616/fd/49 /var/lib/kubelet/pods/2093bbf3-eb23-4139-8076-5bca5f720d1b/volume-subpaths/data/cromwell-runner/0
989841 D /usr/bin/mount -o bind,remount /proc/6616/fd/70 /var/lib/kubelet/pods/322e9748-cb86-4355-9d7d-0e106533cf0a/volume-subpaths/data/cromwell-runner/0
1360682 D sync
1476802 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2303814 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2336314 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2668230 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/rnaseq_eukaryonanalysis/e2ca06ea-5df1-4e0b-83e3-2f0574632829/

zhanglu

strace sync
execve("/usr/bin/sync", ["sync"], 0x7ffe4aa407b0 /* 33 vars /) = 0
brk(NULL) = 0x55f54a1c7000
arch_prctl(0x3001 / ARCH_??? */, 0x7fff30c2dda0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56063, ...}) = 0
mmap(NULL, 56063, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f867bce6000
close(3) = 0
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2405\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=3167872, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f867bce4000
lseek(3, 808, SEEK_SET) = 808
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
mmap(NULL, 3950400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f867b703000
mprotect(0x7f867b8bf000, 2093056, PROT_NONE) = 0
mmap(0x7f867babe000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f867babe000
mmap(0x7f867bac4000, 14144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f867bac4000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7f867bce5540) = 0
mprotect(0x7f867babe000, 16384, PROT_READ) = 0
mprotect(0x55f549b32000, 4096, PROT_READ) = 0
mprotect(0x7f867bcf4000, 4096, PROT_READ) = 0
munmap(0x7f867bce6000, 56063) = 0
brk(NULL) = 0x55f54a1c7000
brk(0x55f54a1e8000) = 0x55f54a1e8000
brk(NULL) = 0x55f54a1e8000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2997, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2997
read(3, "", 4096) = 0
close(3) = 0
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
sync(

zhanglu

ps -eo pid,ppid,state,cmd | grep ^D

zhanglu

timedatectl set-timezone Asia/Shanghai

zhanglu

docker pull dockerhub.genostack.com:8090/library/genostack_kubernetes_terminal:v0.8

docker pull dockerhub.genostack.com:8090/angs/analysis_chemical_shift_table:V03.02

docker pull dockerhub.genostack.com:8090/angs/nmrium:v01

docker pull dockerhub.genostack.com:8090/library/sanshu_database:latest

docker pull dockerhub.genostack.com:8090/library/sanshu_mutant_database:V1

zhanglu

docker save -o prom_alertmanager_v0.23.0.tar prom/alertmanager:v0.23.0

docker save -o kubesphere_prometheus-config-reloader_v0.55.1.tar kubesphere/prometheus-config-reloader:v0.55.1

docker save -o kubesphere_kube-state-metrics_v2.6.0.tar kubesphere/kube-state-metrics:v2.6.0

docker save -o kubesphere_kube-rbac-proxy_v0.11.0.tar kubesphere/kube-rbac-proxy:v0.11.0

docker save -o dockerhub.genostack.com_grafana_12.0.2-security-01-ubuntu.tar dockerhub.genostack.com:8090/library/grafana:12.0.2-security-01-ubuntu

docker save -o prom_node-exporter_v1.3.1.tar prom/node-exporter:v1.3.1

docker save -o kubesphere_notification-manager_v2.3.0.tar kubesphere/notification-manager:v2.3.0

docker save -o kubesphere_notification-tenant-sidecar_v3.2.0.tar kubesphere/notification-tenant-sidecar:v3.2.0

docker save -o kubesphere_notification-manager-operator_v2.3.0.tar kubesphere/notification-manager-operator:v2.3.0

docker save -o prom_prometheus_v2.39.1.tar prom/prometheus:v2.39.1

docker save -o kubesphere_prometheus-operator_v0.55.1.tar kubesphere/prometheus-operator:v0.55.1

docker save -o thanosio_thanos_v0.31.0.tar thanosio/thanos:v0.31.0

zhanglu

c-48ed-b0fe-6af0e10bfe1a")
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.719980 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data podName:404b6cca-8dec-48ed-b0fe-6af0e10bfe1a nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.71995658 +0800 CST m=+85903.723576802 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data") pod "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a" (UID: "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a") : error processing /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.720008 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data podName:599e5d21-3c5f-4898-a87a-38bf7e15e8f3 nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.719990596 +0800 CST m=+85903.723610810 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data") pod "599e5d21-3c5f-4898-a87a-38bf7e15e8f3" (UID: "599e5d21-3c5f-4898-a87a-38bf7e15e8f3") : error processing /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"

zhanglu

复制日志里的挂载路径，查询占用进程

lsof | grep "/var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0"
lsof | grep "/var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0"

zhanglu

Check for open files in that subpath

sudo lsof /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

OR use fuser to see PIDs

sudo fuser -v /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

sudo umount -l /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

root 92467 1 0 11:22 ? 00:00:00 /usr/bin/mount -o bind,remount /proc/3202/fd/48 /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

dmesg -T | grep -i "blocked for more than 120 seconds"

zhanglu

Run ceph -s to check general health.

Run ceph tell mds.* dump captures to see if there are ongoing slow requests or if an MDS is "laggy."

zhanglu

[root@node2 0]# mount | grep ceph
tmpfs on /var/lib/kubelet/pods/9d7dc27a-6ee9-4c85-bc86-a9df68a7b559/volumes/kubernetes.io~secret/rook-csi-cephfs-plugin-sa-token-pb2sn type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/f8dd6c09-38de-4009-a3fb-8c4b5f35e009/volumes/kubernetes.io~projected/ceph-csi-configs type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/371a1859-be44-4db2-b345-642b8c27f6de/volumes/kubernetes.io~secret/rook-ceph-mds-myfs-ec-a-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/e4938d1d-37bb-45cd-97a3-ae836c7f9e25/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/326f83ec-27bb-4b20-8ef5-df9f575f0b91/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ae25aa0a-fbf3-4786-b457-cdaddd158305/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/e4938d1d-37bb-45cd-97a3-ae836c7f9e25/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/37b1b062-36dc-4653-94bc-d65eb9471be5/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ee81bbf3-7678-40e4-8bc7-a273ed4161db/volumes/kubernetes.io~secret/rook-ceph-crash-collector-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/2cf5bdfa-1592-4e7a-89ec-4282784d8c8d/volumes/kubernetes.io~secret/rook-ceph-mons-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/bcd70180-7a0b-45bc-a730-3370b421cda2/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/37b1b062-36dc-4653-94bc-d65eb9471be5/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/01448345-5e05-400a-a5bc-00665b06534d/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ae25aa0a-fbf3-4786-b457-cdaddd158305/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/326f83ec-27bb-4b20-8ef5-df9f575f0b91/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/bcd70180-7a0b-45bc-a730-3370b421cda2/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/af822db5-cdc0-4d81-9ec9-6a9a00229d19/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/01448345-5e05-400a-a5bc-00665b06534d/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/af822db5-cdc0-4d81-9ec9-6a9a00229d19/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/volumes/csi/csi-vol-42070b59-05af-11ee-a677-b62e185b651a/8d7c3218-4807-499d-bc93-bfff24ceb882 on /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-23b68c83-5c16-44a2-8d78-71ea15a1dff3/globalmount type ceph (rw,relatime,name=csi-cephfs-node,secret=<hidden>,fsid=00000000-0000-0000-0000-000000000000,acl,mds_namespace=myfs-ec,_netdev)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/volumes/csi/csi-vol-42070b59-05af-11ee-a677-b62e185b651a/8d7c3218-4807-499d-bc93-bfff24ceb882 on /var/lib/kubelet/pods/12afe02d-0149-491d-8707-ff8ea97af49f/volumes/kubernetes.io~csi/pvc-23b68c83-5c16-44a2-8d78-71ea15a1dff3/mount type ceph (rw,relatime,name=csi-cephfs-node,secret=<hidden>,fsid=00000000-0000-0000-0000-000000000000,acl,mds_namespace=myfs-ec,_netdev)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /cephfs_data type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/ee4a50ef-e31b-439b-b1e1-200aa6527c0a/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/ee4a50ef-e31b-439b-b1e1-200aa6527c0a/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0fb12d38-0ea7-4eab-81c1-44003a0ee4a2/volume-subpaths/data/cromwell-summarizer/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0fb12d38-0ea7-4eab-81c1-44003a0ee4a2/volume-subpaths/data/cromwell-summarizer/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/515a95d8-db33-48b3-82a9-2a3a24a69d5f/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/515a95d8-db33-48b3-82a9-2a3a24a69d5f/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/111df690-fc38-4db1-a2e5-70447f9e80ae/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/111df690-fc38-4db1-a2e5-70447f9e80ae/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/e439a245-ea34-4797-b4b9-3517f9d62650/volume-subpaths/data/websocket/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/a127b7f9-38df-457d-9227-9bab7725f86c/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/a127b7f9-38df-457d-9227-9bab7725f86c/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3b020b6c-7511-486a-8c05-19d719989515/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3b020b6c-7511-486a-8c05-19d719989515/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a6e427f-03f7-4496-b66e-9ad23cb80d67/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a6e427f-03f7-4496-b66e-9ad23cb80d67/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3926f598-a2a7-4f09-a6a7-64bebb7bb96d/volume-subpaths/data/phpfpm/6 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/cc6add18-2573-4a3f-9635-3e6a0cb18508/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/379b60cd-4173-4db2-ae42-6df3f6ff4b5a/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/97aff715-f0b3-4bd9-b6ef-7630cd9a7cdd/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/017ecee2-c97b-4e13-9529-1a7d39e449e9/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/14656dfa-19fb-4ade-8472-7ded179ccec9/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/9f5ea36b-8596-4ae8-a143-0466fa418705/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a328296-e0f5-4e7c-8def-d571becb3f9b/volume-subpaths/data/ms-hippo-flow/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/c4b9e777-73b0-4f48-b73b-6f851e1a133a/volume-subpaths/data/ms-hippo-flow/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0e905f16-fb54-4960-a814-8788a7fa1a5e/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/aa0880e7-5b3c-4608-9769-3bc08473f00f/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/9e5545a5-5a4f-4e1e-a2e0-fc17a16783e5/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/067e33e7-1364-48d8-ab7e-1045d791170d/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)

zhanglu

ceph tell mds.myfs-ec session ls
ceph tell mds.myfs-ec client evict id=<client-id>

zhanglu

mds.myfs-ec-a.session.log mds.myfs-ec-c.session.log mds.myfs-ec-f.session.log