暗能星系

zhanglu

zhanglu

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '9992a98d-9fbc-48a5-bf0f-13af49246128' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'ec7b2a76-ae8a-45f9-b484-5346b6f74efd' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f44f2c67-22bd-4953-92d9-8b2da5207a55' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '346c5771-8908-43b9-a037-416d98f0f8f8' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'af81e1d7-c5e9-4852-b60d-a66196d37705' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f3aaa31b-59f6-4e35-930d-5fdc96e69960' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a9fb5224-ede5-4227-a755-dae75c5e0ffc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e1d59bbb-a51e-471c-9efb-50d4dbb5affc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3ed05f3a-1370-457a-9af7-8f70963afe6d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3192edd4-c3cd-4b56-84c4-d78c80b64287' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a61c1d9b-86b7-4455-85a7-e5435ac57ab5' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2ee83a60-3d6e-4ea8-8616-18bff20e1cb0' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b853a6ea-f7a8-4b14-987b-7152373151ec' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '23fbc5cb-be12-4e80-8510-ab1a10774675' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '8ebdadd1-a61d-419e-9ce0-80b8cac41e9c' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6da5292a-7d2f-40a8-aea3-d4e1b50052b6' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '20283bc4-aacd-47bb-8c73-f450ea432bd7' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e2ca06ea-5df1-4e0b-83e3-2f0574632829' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e6f72645-4253-4581-85bf-e4cda8f51a13' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

670765 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/metage_megahit/9992a98d-9fbc-48a5-bf0f-13af49246128/
832494 D /usr/bin/mount -o bind,remount /proc/6616/fd/62 /var/lib/kubelet/pods/61ee2602-a811-4953-8fbe-c4f1afca762a/volume-subpaths/data/cromwell-runner/0
833804 D /usr/bin/mount -o bind,remount /proc/6616/fd/49 /var/lib/kubelet/pods/2093bbf3-eb23-4139-8076-5bca5f720d1b/volume-subpaths/data/cromwell-runner/0
989841 D /usr/bin/mount -o bind,remount /proc/6616/fd/70 /var/lib/kubelet/pods/322e9748-cb86-4355-9d7d-0e106533cf0a/volume-subpaths/data/cromwell-runner/0
1360682 D sync
1476802 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2303814 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2336314 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2668230 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/rnaseq_eukaryonanalysis/e2ca06ea-5df1-4e0b-83e3-2f0574632829/

zhanglu

strace sync
execve("/usr/bin/sync", ["sync"], 0x7ffe4aa407b0 /* 33 vars /) = 0
brk(NULL) = 0x55f54a1c7000
arch_prctl(0x3001 / ARCH_??? */, 0x7fff30c2dda0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56063, ...}) = 0
mmap(NULL, 56063, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f867bce6000
close(3) = 0
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2405\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=3167872, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f867bce4000
lseek(3, 808, SEEK_SET) = 808
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
mmap(NULL, 3950400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f867b703000
mprotect(0x7f867b8bf000, 2093056, PROT_NONE) = 0
mmap(0x7f867babe000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f867babe000
mmap(0x7f867bac4000, 14144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f867bac4000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7f867bce5540) = 0
mprotect(0x7f867babe000, 16384, PROT_READ) = 0
mprotect(0x55f549b32000, 4096, PROT_READ) = 0
mprotect(0x7f867bcf4000, 4096, PROT_READ) = 0
munmap(0x7f867bce6000, 56063) = 0
brk(NULL) = 0x55f54a1c7000
brk(0x55f54a1e8000) = 0x55f54a1e8000
brk(NULL) = 0x55f54a1e8000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2997, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2997
read(3, "", 4096) = 0
close(3) = 0
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
sync(

zhanglu

ps -eo pid,ppid,state,cmd | grep ^D

zhanglu

timedatectl set-timezone Asia/Shanghai

zhanglu

docker pull dockerhub.genostack.com:8090/library/genostack_kubernetes_terminal:v0.8

docker pull dockerhub.genostack.com:8090/angs/analysis_chemical_shift_table:V03.02

docker pull dockerhub.genostack.com:8090/angs/nmrium:v01

docker pull dockerhub.genostack.com:8090/library/sanshu_database:latest

docker pull dockerhub.genostack.com:8090/library/sanshu_mutant_database:V1

zhanglu

docker save -o prom_alertmanager_v0.23.0.tar prom/alertmanager:v0.23.0

docker save -o kubesphere_prometheus-config-reloader_v0.55.1.tar kubesphere/prometheus-config-reloader:v0.55.1

docker save -o kubesphere_kube-state-metrics_v2.6.0.tar kubesphere/kube-state-metrics:v2.6.0

docker save -o kubesphere_kube-rbac-proxy_v0.11.0.tar kubesphere/kube-rbac-proxy:v0.11.0

docker save -o dockerhub.genostack.com_grafana_12.0.2-security-01-ubuntu.tar dockerhub.genostack.com:8090/library/grafana:12.0.2-security-01-ubuntu

docker save -o prom_node-exporter_v1.3.1.tar prom/node-exporter:v1.3.1

docker save -o kubesphere_notification-manager_v2.3.0.tar kubesphere/notification-manager:v2.3.0

docker save -o kubesphere_notification-tenant-sidecar_v3.2.0.tar kubesphere/notification-tenant-sidecar:v3.2.0

docker save -o kubesphere_notification-manager-operator_v2.3.0.tar kubesphere/notification-manager-operator:v2.3.0

docker save -o prom_prometheus_v2.39.1.tar prom/prometheus:v2.39.1

docker save -o kubesphere_prometheus-operator_v0.55.1.tar kubesphere/prometheus-operator:v0.55.1

docker save -o thanosio_thanos_v0.31.0.tar thanosio/thanos:v0.31.0

zhanglu

c-48ed-b0fe-6af0e10bfe1a")
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.719980 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data podName:404b6cca-8dec-48ed-b0fe-6af0e10bfe1a nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.71995658 +0800 CST m=+85903.723576802 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data") pod "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a" (UID: "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a") : error processing /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.720008 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data podName:599e5d21-3c5f-4898-a87a-38bf7e15e8f3 nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.719990596 +0800 CST m=+85903.723610810 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data") pod "599e5d21-3c5f-4898-a87a-38bf7e15e8f3" (UID: "599e5d21-3c5f-4898-a87a-38bf7e15e8f3") : error processing /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"

zhanglu

复制日志里的挂载路径，查询占用进程

lsof | grep "/var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0"
lsof | grep "/var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0"

zhanglu

Check for open files in that subpath

sudo lsof /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

OR use fuser to see PIDs

sudo fuser -v /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

sudo umount -l /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

root 92467 1 0 11:22 ? 00:00:00 /usr/bin/mount -o bind,remount /proc/3202/fd/48 /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

dmesg -T | grep -i "blocked for more than 120 seconds"

zhanglu

Run ceph -s to check general health.

Run ceph tell mds.* dump captures to see if there are ongoing slow requests or if an MDS is "laggy."

zhanglu

[root@node2 0]# mount | grep ceph
tmpfs on /var/lib/kubelet/pods/9d7dc27a-6ee9-4c85-bc86-a9df68a7b559/volumes/kubernetes.io~secret/rook-csi-cephfs-plugin-sa-token-pb2sn type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/f8dd6c09-38de-4009-a3fb-8c4b5f35e009/volumes/kubernetes.io~projected/ceph-csi-configs type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/371a1859-be44-4db2-b345-642b8c27f6de/volumes/kubernetes.io~secret/rook-ceph-mds-myfs-ec-a-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/e4938d1d-37bb-45cd-97a3-ae836c7f9e25/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/326f83ec-27bb-4b20-8ef5-df9f575f0b91/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ae25aa0a-fbf3-4786-b457-cdaddd158305/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/e4938d1d-37bb-45cd-97a3-ae836c7f9e25/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/37b1b062-36dc-4653-94bc-d65eb9471be5/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ee81bbf3-7678-40e4-8bc7-a273ed4161db/volumes/kubernetes.io~secret/rook-ceph-crash-collector-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/2cf5bdfa-1592-4e7a-89ec-4282784d8c8d/volumes/kubernetes.io~secret/rook-ceph-mons-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/bcd70180-7a0b-45bc-a730-3370b421cda2/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/37b1b062-36dc-4653-94bc-d65eb9471be5/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/01448345-5e05-400a-a5bc-00665b06534d/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/ae25aa0a-fbf3-4786-b457-cdaddd158305/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/326f83ec-27bb-4b20-8ef5-df9f575f0b91/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/bcd70180-7a0b-45bc-a730-3370b421cda2/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/af822db5-cdc0-4d81-9ec9-6a9a00229d19/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/01448345-5e05-400a-a5bc-00665b06534d/volumes/kubernetes.io~secret/rook-ceph-osd-token-rs9p7 type tmpfs (rw,relatime)
tmpfs on /var/lib/kubelet/pods/af822db5-cdc0-4d81-9ec9-6a9a00229d19/volumes/kubernetes.io~secret/rook-ceph-admin-keyring type tmpfs (rw,relatime)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/volumes/csi/csi-vol-42070b59-05af-11ee-a677-b62e185b651a/8d7c3218-4807-499d-bc93-bfff24ceb882 on /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-23b68c83-5c16-44a2-8d78-71ea15a1dff3/globalmount type ceph (rw,relatime,name=csi-cephfs-node,secret=<hidden>,fsid=00000000-0000-0000-0000-000000000000,acl,mds_namespace=myfs-ec,_netdev)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/volumes/csi/csi-vol-42070b59-05af-11ee-a677-b62e185b651a/8d7c3218-4807-499d-bc93-bfff24ceb882 on /var/lib/kubelet/pods/12afe02d-0149-491d-8707-ff8ea97af49f/volumes/kubernetes.io~csi/pvc-23b68c83-5c16-44a2-8d78-71ea15a1dff3/mount type ceph (rw,relatime,name=csi-cephfs-node,secret=<hidden>,fsid=00000000-0000-0000-0000-000000000000,acl,mds_namespace=myfs-ec,_netdev)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /cephfs_data type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/ee4a50ef-e31b-439b-b1e1-200aa6527c0a/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/ee4a50ef-e31b-439b-b1e1-200aa6527c0a/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0fb12d38-0ea7-4eab-81c1-44003a0ee4a2/volume-subpaths/data/cromwell-summarizer/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0fb12d38-0ea7-4eab-81c1-44003a0ee4a2/volume-subpaths/data/cromwell-summarizer/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/515a95d8-db33-48b3-82a9-2a3a24a69d5f/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/515a95d8-db33-48b3-82a9-2a3a24a69d5f/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/111df690-fc38-4db1-a2e5-70447f9e80ae/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/111df690-fc38-4db1-a2e5-70447f9e80ae/volume-subpaths/data/cromwell-frontend/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/e439a245-ea34-4797-b4b9-3517f9d62650/volume-subpaths/data/websocket/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/a127b7f9-38df-457d-9227-9bab7725f86c/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/a127b7f9-38df-457d-9227-9bab7725f86c/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3b020b6c-7511-486a-8c05-19d719989515/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3b020b6c-7511-486a-8c05-19d719989515/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a6e427f-03f7-4496-b66e-9ad23cb80d67/volume-subpaths/data/cromwell-runner/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a6e427f-03f7-4496-b66e-9ad23cb80d67/volume-subpaths/data/cromwell-runner/1 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/3926f598-a2a7-4f09-a6a7-64bebb7bb96d/volume-subpaths/data/phpfpm/6 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/cc6add18-2573-4a3f-9635-3e6a0cb18508/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/379b60cd-4173-4db2-ae42-6df3f6ff4b5a/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/97aff715-f0b3-4bd9-b6ef-7630cd9a7cdd/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/017ecee2-c97b-4e13-9529-1a7d39e449e9/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/14656dfa-19fb-4ade-8472-7ded179ccec9/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/9f5ea36b-8596-4ae8-a143-0466fa418705/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/2a328296-e0f5-4e7c-8def-d571becb3f9b/volume-subpaths/data/ms-hippo-flow/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/c4b9e777-73b0-4f48-b73b-6f851e1a133a/volume-subpaths/data/ms-hippo-flow/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/0e905f16-fb54-4960-a814-8788a7fa1a5e/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/aa0880e7-5b3c-4608-9769-3bc08473f00f/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/9e5545a5-5a4f-4e1e-a2e0-fc17a16783e5/volume-subpaths/data/ms-hippo-flow-new/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)
10.233.18.227:6789,10.233.40.1:6789,10.233.29.234:6789:/ on /var/lib/kubelet/pods/067e33e7-1364-48d8-ab7e-1045d791170d/volume-subpaths/data/cromwell-frontend/0 type ceph (rw,relatime,name=admin,secret=<hidden>,acl)

zhanglu

ceph tell mds.myfs-ec session ls
ceph tell mds.myfs-ec client evict id=<client-id>

zhanglu

mds.myfs-ec-a.session.log mds.myfs-ec-c.session.log mds.myfs-ec-f.session.log

zhanglu

Evict the session on MDS 'f' (Subnet 70.0)

ceph tell mds.myfs-ec-f client evict id=237440910

Evict the session on MDS 'c' (Subnet 108.0)

ceph tell mds.myfs-ec-c client evict id=237445625

Evict the session on MDS 'a' (Subnet 70.0)

ceph tell mds.myfs-ec-a client evict id=237450430

zhanglu

ceph osd df tree | sort -nk 8

zhanglu

bash-4.4$ ceph osd df tree | sort -nk 8
0 hdd 10.69240 0.95000 11 TiB 8.4 TiB 8.3 TiB 6.9 GiB 21 GiB 2.3 TiB 78.25 0.92 58 up osd.0
1 hdd 10.69240 0.95000 11 TiB 9.0 TiB 9.0 TiB 14 GiB 24 GiB 1.7 TiB 84.50 0.99 65 up osd.1
2 hdd 10.69240 0.95000 11 TiB 8.4 TiB 8.4 TiB 6.9 GiB 20 GiB 2.3 TiB 78.40 0.92 62 up osd.2
3 hdd 10.69240 0.95000 11 TiB 8.1 TiB 8.1 TiB 14 GiB 23 GiB 2.6 TiB 75.86 0.89 60 up osd.3
4 hdd 10.69240 0.95000 11 TiB 9.2 TiB 9.2 TiB 6.9 GiB 23 GiB 1.5 TiB 86.06 1.01 65 up osd.4
5 hdd 10.69240 0.95000 11 TiB 8.1 TiB 8.1 TiB 14 GiB 22 GiB 2.6 TiB 75.99 0.89 59 up osd.5
6 hdd 10.69240 0.95000 11 TiB 8.2 TiB 8.2 TiB 6 KiB 18 GiB 2.5 TiB 77.01 0.90 60 up osd.6
7 hdd 10.69240 0.95000 11 TiB 8.8 TiB 8.8 TiB 14 GiB 24 GiB 1.9 TiB 82.50 0.97 67 up osd.7
8 hdd 10.69240 0.95000 11 TiB 9.5 TiB 9.4 TiB 7.3 GiB 23 GiB 1.2 TiB 88.39 1.04 65 up osd.8
9 hdd 10.69240 0.95000 11 TiB 9.0 TiB 8.9 TiB 6.8 GiB 22 GiB 1.7 TiB 83.78 0.98 65 up osd.9
10 hdd 10.69240 0.95000 11 TiB 9.8 TiB 9.7 TiB 8.9 MiB 22 GiB 949 GiB 91.33 1.07 67 up osd.10
11 hdd 10.69240 0.95000 11 TiB 9.3 TiB 9.2 TiB 1 KiB 21 GiB 1.4 TiB 86.60 1.02 65 up osd.11
12 hdd 10.69240 0.95000 11 TiB 9.1 TiB 9.0 TiB 116 KiB 20 GiB 1.6 TiB 84.69 0.99 64 up osd.12
13 hdd 10.69240 0.95000 11 TiB 9.1 TiB 9.1 TiB 7.2 GiB 22 GiB 1.6 TiB 85.26 1.00 65 up osd.13
14 hdd 10.69240 0.95000 11 TiB 9.8 TiB 9.8 TiB 327 KiB 22 GiB 918 GiB 91.61 1.08 63 up osd.14
15 hdd 10.69240 0.95000 11 TiB 9.5 TiB 9.5 TiB 14 GiB 24 GiB 1.1 TiB 89.25 1.05 68 up osd.15
16 hdd 10.69240 0.95000 11 TiB 8.7 TiB 8.6 TiB 1 KiB 20 GiB 2.0 TiB 81.06 0.95 62 up osd.16
17 hdd 10.69240 0.95000 11 TiB 9.4 TiB 9.4 TiB 4.1 MiB 20 GiB 1.3 TiB 87.83 1.03 65 up osd.17
18 hdd 10.69240 0.95000 11 TiB 9.6 TiB 9.6 TiB 14 GiB 25 GiB 1.0 TiB 90.21 1.06 66 up osd.18
19 hdd 10.69240 0.95000 11 TiB 9.0 TiB 9.0 TiB 21 GiB 26 GiB 1.7 TiB 84.22 0.99 65 up osd.19
20 hdd 10.69240 0.95000 11 TiB 9.6 TiB 9.5 TiB 1 KiB 22 GiB 1.1 TiB 89.41 1.05 65 up osd.20
21 hdd 18.19040 1.00000 18 TiB 16 TiB 15 TiB 14 GiB 37 GiB 2.7 TiB 85.40 1.00 114 up osd.21
22 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 7.2 GiB 29 GiB 2.0 TiB 86.54 1.02 90 up osd.22
23 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 1 KiB 27 GiB 2.0 TiB 86.09 1.01 89 up osd.23
24 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 6.9 GiB 31 GiB 1.2 TiB 91.48 1.07 94 up osd.24
25 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 1.5 MiB 27 GiB 2.5 TiB 83.09 0.98 87 up osd.25
26 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 6.9 GiB 30 GiB 1.2 TiB 91.56 1.07 91 up osd.26
27 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 735 KiB 27 GiB 2.2 TiB 84.76 1.00 87 up osd.27
28 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 21 GiB 32 GiB 2.4 TiB 83.82 0.98 87 up osd.28
29 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 7.1 GiB 31 GiB 1.1 TiB 92.15 1.08 93 up osd.29
30 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 14 GiB 30 GiB 2.6 TiB 81.97 0.96 88 up osd.30
31 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 1 KiB 28 GiB 1.6 TiB 88.87 1.04 96 up osd.31
32 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 2.6 MiB 26 GiB 2.9 TiB 80.04 0.94 87 up osd.32
33 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 14 GiB 30 GiB 2.5 TiB 83.05 0.98 91 up osd.33
34 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 6.9 GiB 29 GiB 2.1 TiB 85.82 1.01 91 up osd.34
35 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 14 GiB 30 GiB 2.2 TiB 84.75 1.00 89 up osd.35
36 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 1 KiB 28 GiB 1.7 TiB 88.45 1.04 94 up osd.36
37 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 767 KiB 27 GiB 1.9 TiB 86.76 1.02 90 up osd.37
38 hdd 14.55269 1.00000 15 TiB 12 TiB 12 TiB 21 MiB 26 GiB 2.2 TiB 84.96 1.00 88 up osd.38
39 hdd 14.55269 1.00000 15 TiB 13 TiB 13 TiB 467 KiB 28 GiB 1.5 TiB 89.62 1.05 90 up osd.39
40 hdd 14.55269 1.00000 15 TiB 13 TiB 12 TiB 14 GiB 30 GiB 2.0 TiB 86.04 1.01 93 up osd.40
41 hdd 18.19040 1.00000 18 TiB 16 TiB 16 TiB 14 GiB 37 GiB 2.5 TiB 86.19 1.01 110 up osd.41
42 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 7.2 GiB 35 GiB 3.1 TiB 82.81 0.97 110 up osd.42
43 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 1 KiB 33 GiB 2.7 TiB 85.10 1.00 113 up osd.43
44 hdd 18.19040 1.00000 18 TiB 16 TiB 16 TiB 14 GiB 38 GiB 2.5 TiB 86.12 1.01 106 up osd.44
45 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 1 KiB 33 GiB 3.1 TiB 82.99 0.97 104 up osd.45
46 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 14 GiB 36 GiB 3.1 TiB 83.14 0.98 106 up osd.46
47 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 6.8 GiB 35 GiB 3.0 TiB 83.41 0.98 106 up osd.47
48 hdd 18.19040 1.00000 18 TiB 17 TiB 17 TiB 2.2 MiB 36 GiB 1.5 TiB 91.72 1.08 110 up osd.48
49 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 7.1 GiB 33 GiB 3.6 TiB 80.43 0.94 105 up osd.49
50 hdd 18.19040 1.00000 18 TiB 15 TiB 15 TiB 7 KiB 32 GiB 3.3 TiB 81.94 0.96 103 up osd.50
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
MIN/MAX VAR: 0.89/1.08 STDDEV: 4.04
-3 74.84679 - 75 TiB 59 TiB 59 TiB 63 GiB 151 GiB 15 TiB 79.44 0.93 - host node1
-5 74.84679 - 75 TiB 64 TiB 64 TiB 35 GiB 154 GiB 10 TiB 86.08 1.01 - host node2
-9 72.76344 - 73 TiB 64 TiB 64 TiB 21 GiB 144 GiB 8.9 TiB 87.75 1.03 - host node7
-7 74.84679 - 75 TiB 66 TiB 65 TiB 49 GiB 159 GiB 9.2 TiB 87.66 1.03 - host node3
-11 101.86882 - 102 TiB 87 TiB 86 TiB 56 GiB 203 GiB 15 TiB 84.95 1.00 - host node5
-13 101.86882 - 102 TiB 88 TiB 88 TiB 35 GiB 197 GiB 14 TiB 86.63 1.02 - host node6
-15 200.09439 - 200 TiB 169 TiB 169 TiB 77 GiB 385 GiB 31 TiB 84.48 0.99 - host node8
TOTAL 701 TiB 597 TiB 595 TiB 335 GiB 1.4 TiB 104 TiB 85.17
-1 701.13586 - 701 TiB 597 TiB 595 TiB 335 GiB 1.4 TiB 104 TiB 85.17 1.00 - root default