暗能星系

zhanglu

6e362360-32d0-4311-8d9f-a1a2ed5b61c2
e845ff2e-95ba-4108-bec5-7b8ece2fe76d
e1ae68d6-8c6d-4f91-b970-06e65e57c564

zhanglu

select "WORKFLOW_EXECUTION_UUID" , "WORKFLOW_NAME", "WORKFLOW_STATUS" from "WORKFLOW_METADATA_SUMMARY_ENTRY" where "WORKFLOW_STATUS" = 'Running';

zhanglu

[root@node3 cephfs_data]# ps -eo pid,stat,cmd | grep -E '^.+ D'
647454 D sync
773495 D sync
777729 D /usr/bin/mount -o bind,remount /proc/7817/fd/50 /var/lib/kubelet/pods/ce2112de-6092-400b-8d1f-ff9c7f0feb60/volume-subpaths/data/cromwell/0
807143 D sync
831541 D sync
862778 D sync
923295 D sync
925182 D sync
1120488 D sync
1137925 D sync
1438346 D sync
1685722 D /usr/bin/mount -o bind,remount /proc/7817/fd/54 /var/lib/kubelet/pods/01c428af-c57a-4159-93ce-4e57c73ed85b/volume-subpaths/data/cromwell/0
1698130 D sync
1699192 D sync
1729571 D sync
1729918 D sync
1916667 D sync
1957175 D sync
3327874 S+ grep --color=auto -E ^.+ D
3402901 DNs (coredump)
3883466 D+ rm -rf ./5b9c9af2-49ab-4cd3-bbe9-637c131ef344 ./ff853c49-720c-442a-b5bf-fb50c63736c2 ./4dba7671-4615-4ce8-8083-09c1b6a3b895 ./a00fd097-cd21-4561-9dcc-2bb9b66d58fa ./2ca5d416-7398-43f4-b368-68e790abaf95 ./f7f4938d-0b2a-4e95-9f6c-0b9e3c442ef6 ./5958823a-cb9b-4556-a30d-4142eba696c7 ./8281f2d8-fcb9-4508-853d-be96fe10535e ./0ee9f259-014c-44e1-9453-f2af6f423c06 ./4fbe43e5-4018-434f-a6fd-61ba3ebc69e6 ./03a22cb6-5ca6-4b58-8f32-b95988a4605c ./58ade6fd-845a-4ee5-bebd-b0dce6cfdac5 ./4b8fde55-6387-4fd3-9092-d4c19c44ab9c ./aed461b9-dfff-4fc1-b089-f33b2673b1c5 ./7a932a56-0a47-42d4-893b-d0435a2edc1c ./57ff22a3-5fa4-4d77-94b7-71011cb6238e ./e8c72d7c-1d92-4d3c-8702-f4cf50ea2ee2 ./93a9de1b-7be3-40ba-9e9e-3364b67b202d ./e3d77188-3693-4f7f-81e7-8688b4b01dec ./8bf3777b-361c-46c1-9328-c9f3d5cc2822 ./eaf418dc-01b3-4bc0-b272-d2ba3f2493ce ./1c24a9f4-2e97-44c8-af0a-85db7e374956 ./77b44fc3-2f4b-480b-b01a-d57a1395fe1e ./f2fd7fe2-3903-437b-a40f-2e812dc35201 ./a862e7a9-99e2-4c31-82ba-3e87c0421354 ./b16de460-9245-41d5-8b69-444e95e549a0 ./815801ee-1ded-46db-933a-0decd35d5fe4 ./75f6352d-2080-40dc-9ac5-ce4c6566e0c2 ./9ef6a0eb-1d37-4966-b7e1-1ffc4b9abc22 ./d97f36bd-6805-4bfa-a386-0b90bb42d8cd ./345b403e-45e2-4e09-9b99-23b1b3c7f33c ./c5ef2a0e-2868-4e09-9c26-cf5357f60448 ./b0b9b6c3-bcb6-479f-83c5-429e306ff1ad ./4ec53f20-4f2c-430c-a332-7736584b26c7 ./02ac4847-d640-4312-a761-e7fd01d45c27 ./53630622-fa7f-4cf6-91b2-12768883af6f ./e1660f00-841f-4899-a4b2-c2451edf2b36 ./07de3f21-1de8-4326-9bf7-5774a2791eaa ./97914a70-2fe4-4c1f-a326-adebe8222df0 ./695f925d-723b-4446-b9b1-11dfa642f065
3911819 D sync
3948171 D sync

zhanglu

SELECT "METADATA_KEY", COUNT()
FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = 'e886888f-c972-4932-a9a4-187525877ea0'
GROUP BY "METADATA_KEY"
ORDER BY COUNT() DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = '32d44941-c654-4163-b01e-aa41a0cc597a'
AND "METADATA_KEY" = 'vmStartTime'

          METADATA_KEY              |  count

zhanglu

e886888f-c972-4932-a9a4-187525877ea0
b9eec03b-590a-4a53-8a99-088fe01c9070
2e97dac5-f31b-4cbe-8712-4610295c6dfa
731bad53-fe54-43a2-b1d1-2639ac371a43

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b9eec03b-590a-4a53-8a99-088fe01c9070' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2e97dac5-f31b-4cbe-8712-4610295c6dfa' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '731bad53-fe54-43a2-b1d1-2639ac371a43' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" in (SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;);

zhanglu

zhanglu

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '9992a98d-9fbc-48a5-bf0f-13af49246128' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'ec7b2a76-ae8a-45f9-b484-5346b6f74efd' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f44f2c67-22bd-4953-92d9-8b2da5207a55' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '346c5771-8908-43b9-a037-416d98f0f8f8' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'af81e1d7-c5e9-4852-b60d-a66196d37705' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f3aaa31b-59f6-4e35-930d-5fdc96e69960' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a9fb5224-ede5-4227-a755-dae75c5e0ffc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e1d59bbb-a51e-471c-9efb-50d4dbb5affc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3ed05f3a-1370-457a-9af7-8f70963afe6d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3192edd4-c3cd-4b56-84c4-d78c80b64287' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a61c1d9b-86b7-4455-85a7-e5435ac57ab5' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2ee83a60-3d6e-4ea8-8616-18bff20e1cb0' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b853a6ea-f7a8-4b14-987b-7152373151ec' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '23fbc5cb-be12-4e80-8510-ab1a10774675' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '8ebdadd1-a61d-419e-9ce0-80b8cac41e9c' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6da5292a-7d2f-40a8-aea3-d4e1b50052b6' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '20283bc4-aacd-47bb-8c73-f450ea432bd7' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e2ca06ea-5df1-4e0b-83e3-2f0574632829' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e6f72645-4253-4581-85bf-e4cda8f51a13' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

670765 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/metage_megahit/9992a98d-9fbc-48a5-bf0f-13af49246128/
832494 D /usr/bin/mount -o bind,remount /proc/6616/fd/62 /var/lib/kubelet/pods/61ee2602-a811-4953-8fbe-c4f1afca762a/volume-subpaths/data/cromwell-runner/0
833804 D /usr/bin/mount -o bind,remount /proc/6616/fd/49 /var/lib/kubelet/pods/2093bbf3-eb23-4139-8076-5bca5f720d1b/volume-subpaths/data/cromwell-runner/0
989841 D /usr/bin/mount -o bind,remount /proc/6616/fd/70 /var/lib/kubelet/pods/322e9748-cb86-4355-9d7d-0e106533cf0a/volume-subpaths/data/cromwell-runner/0
1360682 D sync
1476802 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2303814 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2336314 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2668230 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/rnaseq_eukaryonanalysis/e2ca06ea-5df1-4e0b-83e3-2f0574632829/

zhanglu

strace sync
execve("/usr/bin/sync", ["sync"], 0x7ffe4aa407b0 /* 33 vars /) = 0
brk(NULL) = 0x55f54a1c7000
arch_prctl(0x3001 / ARCH_??? */, 0x7fff30c2dda0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56063, ...}) = 0
mmap(NULL, 56063, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f867bce6000
close(3) = 0
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2405\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=3167872, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f867bce4000
lseek(3, 808, SEEK_SET) = 808
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
mmap(NULL, 3950400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f867b703000
mprotect(0x7f867b8bf000, 2093056, PROT_NONE) = 0
mmap(0x7f867babe000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f867babe000
mmap(0x7f867bac4000, 14144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f867bac4000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7f867bce5540) = 0
mprotect(0x7f867babe000, 16384, PROT_READ) = 0
mprotect(0x55f549b32000, 4096, PROT_READ) = 0
mprotect(0x7f867bcf4000, 4096, PROT_READ) = 0
munmap(0x7f867bce6000, 56063) = 0
brk(NULL) = 0x55f54a1c7000
brk(0x55f54a1e8000) = 0x55f54a1e8000
brk(NULL) = 0x55f54a1e8000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2997, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2997
read(3, "", 4096) = 0
close(3) = 0
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
sync(

zhanglu

ps -eo pid,ppid,state,cmd | grep ^D

zhanglu

timedatectl set-timezone Asia/Shanghai

zhanglu

docker pull dockerhub.genostack.com:8090/library/genostack_kubernetes_terminal:v0.8

docker pull dockerhub.genostack.com:8090/angs/analysis_chemical_shift_table:V03.02

docker pull dockerhub.genostack.com:8090/angs/nmrium:v01

docker pull dockerhub.genostack.com:8090/library/sanshu_database:latest

docker pull dockerhub.genostack.com:8090/library/sanshu_mutant_database:V1

zhanglu

docker save -o prom_alertmanager_v0.23.0.tar prom/alertmanager:v0.23.0

docker save -o kubesphere_prometheus-config-reloader_v0.55.1.tar kubesphere/prometheus-config-reloader:v0.55.1

docker save -o kubesphere_kube-state-metrics_v2.6.0.tar kubesphere/kube-state-metrics:v2.6.0

docker save -o kubesphere_kube-rbac-proxy_v0.11.0.tar kubesphere/kube-rbac-proxy:v0.11.0

docker save -o dockerhub.genostack.com_grafana_12.0.2-security-01-ubuntu.tar dockerhub.genostack.com:8090/library/grafana:12.0.2-security-01-ubuntu

docker save -o prom_node-exporter_v1.3.1.tar prom/node-exporter:v1.3.1

docker save -o kubesphere_notification-manager_v2.3.0.tar kubesphere/notification-manager:v2.3.0

docker save -o kubesphere_notification-tenant-sidecar_v3.2.0.tar kubesphere/notification-tenant-sidecar:v3.2.0

docker save -o kubesphere_notification-manager-operator_v2.3.0.tar kubesphere/notification-manager-operator:v2.3.0

docker save -o prom_prometheus_v2.39.1.tar prom/prometheus:v2.39.1

docker save -o kubesphere_prometheus-operator_v0.55.1.tar kubesphere/prometheus-operator:v0.55.1

docker save -o thanosio_thanos_v0.31.0.tar thanosio/thanos:v0.31.0

zhanglu

c-48ed-b0fe-6af0e10bfe1a")
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.719980 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data podName:404b6cca-8dec-48ed-b0fe-6af0e10bfe1a nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.71995658 +0800 CST m=+85903.723576802 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a-data") pod "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a" (UID: "404b6cca-8dec-48ed-b0fe-6af0e10bfe1a") : error processing /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"
5月 13 13:51:39 node2 kubelet[3202]: E0513 13:51:39.720008 3202 nestedpendingoperations.go:301] Operation for "{volumeName:kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data podName:599e5d21-3c5f-4898-a87a-38bf7e15e8f3 nodeName:}" failed. No retries permitted until 2026-05-13 13:53:41.719990596 +0800 CST m=+85903.723610810 (durationBeforeRetry 2m2s). Error: "error cleaning subPath mounts for volume "data" (UniqueName: "kubernetes.io/host-path/599e5d21-3c5f-4898-a87a-38bf7e15e8f3-data") pod "599e5d21-3c5f-4898-a87a-38bf7e15e8f3" (UID: "599e5d21-3c5f-4898-a87a-38bf7e15e8f3") : error processing /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new: error cleaning subpath mount /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: unmount failed: exit status 32\nUnmounting arguments: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0\nOutput: umount: /var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0: target is busy.\n"

zhanglu

复制日志里的挂载路径，查询占用进程

lsof | grep "/var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0"
lsof | grep "/var/lib/kubelet/pods/599e5d21-3c5f-4898-a87a-38bf7e15e8f3/volume-subpaths/data/ms-hippo-flow-new/0"

zhanglu

Check for open files in that subpath

sudo lsof /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

OR use fuser to see PIDs

sudo fuser -v /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

sudo umount -l /var/lib/kubelet/pods/404b6cca-8dec-48ed-b0fe-6af0e10bfe1a/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

root 92467 1 0 11:22 ? 00:00:00 /usr/bin/mount -o bind,remount /proc/3202/fd/48 /var/lib/kubelet/pods/80dbf45b-924f-4fbf-9ac5-b903580798a0/volume-subpaths/data/ms-hippo-flow-new/0

zhanglu

dmesg -T | grep -i "blocked for more than 120 seconds"

zhanglu

Run ceph -s to check general health.

Run ceph tell mds.* dump captures to see if there are ongoing slow requests or if an MDS is "laggy."

tmp

复制日志里的挂载路径，查询占用进程

Check for open files in that subpath

OR use fuser to see PIDs