暗能星系

zhanglu

[root@node1 prod]# kubectl get pod -n genostack-base-dev
NAME READY STATUS RESTARTS AGE
postgres-7cffc9f66-6ngfr 0/1 ContainerCreating 0 17m
postgres-7cffc9f66-fqmzm 0/1 Terminating 0 27m
redis-bd7c6cbb4-gtdz9 1/1 Running 0 30d

zhanglu

Resource Requests Limits

cpu 46867m (41%) 58400m (52%)
memory 242859107584 (28%) 315783595264 (37%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
Events:
Type Reason Age From Message

Normal NodeReady 8m (x252 over 16h) kubelet Node node1 status is now: NodeReady
Normal NodeNotReady 59s (x254 over 16h) kubelet Node node1 status is now: NodeNotReady

zhanglu

ps -eo pid,stat,cmd | grep -E '^.+ D'

zhanglu

[root@node1 account]# ps -eo pid,stat,cmd | grep -E '^.+ D'
382513 Ds /opt/gitlab/embedded/bin/postgres -D /var/opt/gitlab/postgresql/data
382518 Ds postgres: autovacuum launcher process
382519 Ds postgres: stats collector process
385554 D svlogd -tt /var/log/gitlab/redis
385557 D svlogd -tt /var/log/gitlab/sidekiq
385704 D svlogd -tt /var/log/gitlab/node-exporter
385705 D svlogd -tt /var/log/gitlab/gitlab-monitor
385707 D svlogd -tt /var/log/gitlab/prometheus
385722 Ds postgres: gitlab-psql postgres [local] SELECT
385926 D tail --follow=name --retry /var/log/gitlab/nginx/error.log /var/log/gitlab/nginx/current /var/log/gitlab/nginx/gitlab_error.log /var/log/gitlab/nginx/access.log /var/log/gitlab/nginx/gitlab_access.log /var/log/gitlab/prometheus/current /var/log/gitlab/prometheus/state /var/log/gitlab/postgresql/current /var/log/gitlab/postgresql/state /var/log/gitlab/redis/current /var/log/gitlab/redis/state /var/log/gitlab/alertmanager/current /var/log/gitlab/alertmanager/state /var/log/gitlab/redis-exporter/current /var/log/gitlab/redis-exporter/state /var/log/gitlab/gitlab-shell/gitlab-shell.log /var/log/gitlab/sidekiq/current /var/log/gitlab/sidekiq/state /var/log/gitlab/gitlab-monitor/current /var/log/gitlab/gitlab-monitor/state /var/log/gitlab/sshd/current /var/log/gitlab/gitlab-workhorse/current /var/log/gitlab/gitlab-workhorse/state /var/log/gitlab/unicorn/unicorn_stderr.log /var/log/gitlab/unicorn/current /var/log/gitlab/unicorn/unicorn_stdout.log /var/log/gitlab/unicorn/state /var/log/gitlab/gitaly/current /var/log/gitlab/gitaly/state /var/log/gitlab/postgres-exporter/current /var/log/gitlab/postgres-exporter/state /var/log/gitlab/node-exporter/current /var/log/gitlab/node-exporter/state /var/log/gitlab/gitlab-rails/production.log /var/log/gitlab/gitlab-rails/gitlab-rails-db-migrate-2023-02-21-09-56-26.log /var/log/gitlab/gitlab-rails/application.log /var/log/gitlab/gitlab-rails/grpc.log /var/log/gitlab/gitlab-rails/production_json.log /var/log/gitlab/gitlab-rails/api_json.log /var/log/gitlab/logrotate/current /var/log/gitlab/logrotate/state

zhanglu

SELECT count(*) FROM "METADATA_ENTRY";
SELECT * FROM "METADATA_ENTRY" where "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' ;
SELECT * FROM "METADATA_ENTRY" where "WORKFLOW_EXECUTION_UUID" = 'e1ae68d6-8c6d-4f91-b970-06e65e57c564' ;

zhanglu

http://192.168.30.202:31237/api/workflows/v1/70d9a7f6-33c9-4ef4-80bf-62e7af9cc662/metadata

zhanglu

6e362360-32d0-4311-8d9f-a1a2ed5b61c2
e845ff2e-95ba-4108-bec5-7b8ece2fe76d
e1ae68d6-8c6d-4f91-b970-06e65e57c564

zhanglu

select "WORKFLOW_EXECUTION_UUID" , "WORKFLOW_NAME", "WORKFLOW_STATUS" from "WORKFLOW_METADATA_SUMMARY_ENTRY" where "WORKFLOW_STATUS" = 'Running';

zhanglu

[root@node3 cephfs_data]# ps -eo pid,stat,cmd | grep -E '^.+ D'
647454 D sync
773495 D sync
777729 D /usr/bin/mount -o bind,remount /proc/7817/fd/50 /var/lib/kubelet/pods/ce2112de-6092-400b-8d1f-ff9c7f0feb60/volume-subpaths/data/cromwell/0
807143 D sync
831541 D sync
862778 D sync
923295 D sync
925182 D sync
1120488 D sync
1137925 D sync
1438346 D sync
1685722 D /usr/bin/mount -o bind,remount /proc/7817/fd/54 /var/lib/kubelet/pods/01c428af-c57a-4159-93ce-4e57c73ed85b/volume-subpaths/data/cromwell/0
1698130 D sync
1699192 D sync
1729571 D sync
1729918 D sync
1916667 D sync
1957175 D sync
3327874 S+ grep --color=auto -E ^.+ D
3402901 DNs (coredump)
3883466 D+ rm -rf ./5b9c9af2-49ab-4cd3-bbe9-637c131ef344 ./ff853c49-720c-442a-b5bf-fb50c63736c2 ./4dba7671-4615-4ce8-8083-09c1b6a3b895 ./a00fd097-cd21-4561-9dcc-2bb9b66d58fa ./2ca5d416-7398-43f4-b368-68e790abaf95 ./f7f4938d-0b2a-4e95-9f6c-0b9e3c442ef6 ./5958823a-cb9b-4556-a30d-4142eba696c7 ./8281f2d8-fcb9-4508-853d-be96fe10535e ./0ee9f259-014c-44e1-9453-f2af6f423c06 ./4fbe43e5-4018-434f-a6fd-61ba3ebc69e6 ./03a22cb6-5ca6-4b58-8f32-b95988a4605c ./58ade6fd-845a-4ee5-bebd-b0dce6cfdac5 ./4b8fde55-6387-4fd3-9092-d4c19c44ab9c ./aed461b9-dfff-4fc1-b089-f33b2673b1c5 ./7a932a56-0a47-42d4-893b-d0435a2edc1c ./57ff22a3-5fa4-4d77-94b7-71011cb6238e ./e8c72d7c-1d92-4d3c-8702-f4cf50ea2ee2 ./93a9de1b-7be3-40ba-9e9e-3364b67b202d ./e3d77188-3693-4f7f-81e7-8688b4b01dec ./8bf3777b-361c-46c1-9328-c9f3d5cc2822 ./eaf418dc-01b3-4bc0-b272-d2ba3f2493ce ./1c24a9f4-2e97-44c8-af0a-85db7e374956 ./77b44fc3-2f4b-480b-b01a-d57a1395fe1e ./f2fd7fe2-3903-437b-a40f-2e812dc35201 ./a862e7a9-99e2-4c31-82ba-3e87c0421354 ./b16de460-9245-41d5-8b69-444e95e549a0 ./815801ee-1ded-46db-933a-0decd35d5fe4 ./75f6352d-2080-40dc-9ac5-ce4c6566e0c2 ./9ef6a0eb-1d37-4966-b7e1-1ffc4b9abc22 ./d97f36bd-6805-4bfa-a386-0b90bb42d8cd ./345b403e-45e2-4e09-9b99-23b1b3c7f33c ./c5ef2a0e-2868-4e09-9c26-cf5357f60448 ./b0b9b6c3-bcb6-479f-83c5-429e306ff1ad ./4ec53f20-4f2c-430c-a332-7736584b26c7 ./02ac4847-d640-4312-a761-e7fd01d45c27 ./53630622-fa7f-4cf6-91b2-12768883af6f ./e1660f00-841f-4899-a4b2-c2451edf2b36 ./07de3f21-1de8-4326-9bf7-5774a2791eaa ./97914a70-2fe4-4c1f-a326-adebe8222df0 ./695f925d-723b-4446-b9b1-11dfa642f065
3911819 D sync
3948171 D sync

zhanglu

SELECT "METADATA_KEY", COUNT()
FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = 'e886888f-c972-4932-a9a4-187525877ea0'
GROUP BY "METADATA_KEY"
ORDER BY COUNT() DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY"
WHERE "WORKFLOW_EXECUTION_UUID" = '32d44941-c654-4163-b01e-aa41a0cc597a'
AND "METADATA_KEY" = 'vmStartTime'

          METADATA_KEY              |  count

zhanglu

e886888f-c972-4932-a9a4-187525877ea0
b9eec03b-590a-4a53-8a99-088fe01c9070
2e97dac5-f31b-4cbe-8712-4610295c6dfa
731bad53-fe54-43a2-b1d1-2639ac371a43

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b9eec03b-590a-4a53-8a99-088fe01c9070' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2e97dac5-f31b-4cbe-8712-4610295c6dfa' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '731bad53-fe54-43a2-b1d1-2639ac371a43' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" in (SELECT
"WORKFLOW_EXECUTION_UUID"
FROM "METADATA_ENTRY"
WHERE "METADATA_KEY" = 'vmStartTime'
GROUP BY "WORKFLOW_EXECUTION_UUID", "METADATA_KEY"
ORDER BY COUNT("METADATA_KEY") DESC
LIMIT 20;);

zhanglu

zhanglu

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '9992a98d-9fbc-48a5-bf0f-13af49246128' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'ec7b2a76-ae8a-45f9-b484-5346b6f74efd' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';

DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f44f2c67-22bd-4953-92d9-8b2da5207a55' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '346c5771-8908-43b9-a037-416d98f0f8f8' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'af81e1d7-c5e9-4852-b60d-a66196d37705' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6e362360-32d0-4311-8d9f-a1a2ed5b61c2' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'f3aaa31b-59f6-4e35-930d-5fdc96e69960' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a9fb5224-ede5-4227-a755-dae75c5e0ffc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'd0bc517a-5617-44c0-a054-b940fbcf8e86' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e1d59bbb-a51e-471c-9efb-50d4dbb5affc' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3ed05f3a-1370-457a-9af7-8f70963afe6d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '3192edd4-c3cd-4b56-84c4-d78c80b64287' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'a61c1d9b-86b7-4455-85a7-e5435ac57ab5' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e845ff2e-95ba-4108-bec5-7b8ece2fe76d' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '2ee83a60-3d6e-4ea8-8616-18bff20e1cb0' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'b853a6ea-f7a8-4b14-987b-7152373151ec' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '23fbc5cb-be12-4e80-8510-ab1a10774675' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '8ebdadd1-a61d-419e-9ce0-80b8cac41e9c' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '6da5292a-7d2f-40a8-aea3-d4e1b50052b6' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = '20283bc4-aacd-47bb-8c73-f450ea432bd7' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e2ca06ea-5df1-4e0b-83e3-2f0574632829' AND "METADATA_KEY" = 'vmStartTime';
DELETE FROM "METADATA_ENTRY" WHERE "WORKFLOW_EXECUTION_UUID" = 'e6f72645-4253-4581-85bf-e4cda8f51a13' AND "METADATA_KEY" = 'vmStartTime';

zhanglu

670765 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/metage_megahit/9992a98d-9fbc-48a5-bf0f-13af49246128/
832494 D /usr/bin/mount -o bind,remount /proc/6616/fd/62 /var/lib/kubelet/pods/61ee2602-a811-4953-8fbe-c4f1afca762a/volume-subpaths/data/cromwell-runner/0
833804 D /usr/bin/mount -o bind,remount /proc/6616/fd/49 /var/lib/kubelet/pods/2093bbf3-eb23-4139-8076-5bca5f720d1b/volume-subpaths/data/cromwell-runner/0
989841 D /usr/bin/mount -o bind,remount /proc/6616/fd/70 /var/lib/kubelet/pods/322e9748-cb86-4355-9d7d-0e106533cf0a/volume-subpaths/data/cromwell-runner/0
1360682 D sync
1476802 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2303814 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2336314 Dl /usr/local/envs/py310/bin/python /usr/local/envs/py310/bin/gunicorn wsgi:flask_app -c instance/gunicorn.conf.py
2668230 D rm -rf /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/rnaseq_eukaryonanalysis/e2ca06ea-5df1-4e0b-83e3-2f0574632829/

zhanglu

strace sync
execve("/usr/bin/sync", ["sync"], 0x7ffe4aa407b0 /* 33 vars /) = 0
brk(NULL) = 0x55f54a1c7000
arch_prctl(0x3001 / ARCH_??? */, 0x7fff30c2dda0) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56063, ...}) = 0
mmap(NULL, 56063, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f867bce6000
close(3) = 0
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2405\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=3167872, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f867bce4000
lseek(3, 808, SEEK_SET) = 808
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
mmap(NULL, 3950400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f867b703000
mprotect(0x7f867b8bf000, 2093056, PROT_NONE) = 0
mmap(0x7f867babe000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f867babe000
mmap(0x7f867bac4000, 14144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f867bac4000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7f867bce5540) = 0
mprotect(0x7f867babe000, 16384, PROT_READ) = 0
mprotect(0x55f549b32000, 4096, PROT_READ) = 0
mprotect(0x7f867bcf4000, 4096, PROT_READ) = 0
munmap(0x7f867bce6000, 56063) = 0
brk(NULL) = 0x55f54a1c7000
brk(0x55f54a1e8000) = 0x55f54a1e8000
brk(NULL) = 0x55f54a1e8000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2997, ...}) = 0
read(3, "# Locale name alias data base.\n#"..., 4096) = 2997
read(3, "", 4096) = 0
close(3) = 0
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh_CN/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/zh/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
sync(

zhanglu

ps -eo pid,ppid,state,cmd | grep ^D

zhanglu

timedatectl set-timezone Asia/Shanghai

zhanglu

docker pull dockerhub.genostack.com:8090/library/genostack_kubernetes_terminal:v0.8

docker pull dockerhub.genostack.com:8090/angs/analysis_chemical_shift_table:V03.02

docker pull dockerhub.genostack.com:8090/angs/nmrium:v01

docker pull dockerhub.genostack.com:8090/library/sanshu_database:latest

docker pull dockerhub.genostack.com:8090/library/sanshu_mutant_database:V1

zhanglu

docker save -o prom_alertmanager_v0.23.0.tar prom/alertmanager:v0.23.0

docker save -o kubesphere_prometheus-config-reloader_v0.55.1.tar kubesphere/prometheus-config-reloader:v0.55.1

docker save -o kubesphere_kube-state-metrics_v2.6.0.tar kubesphere/kube-state-metrics:v2.6.0

docker save -o kubesphere_kube-rbac-proxy_v0.11.0.tar kubesphere/kube-rbac-proxy:v0.11.0

docker save -o dockerhub.genostack.com_grafana_12.0.2-security-01-ubuntu.tar dockerhub.genostack.com:8090/library/grafana:12.0.2-security-01-ubuntu

docker save -o prom_node-exporter_v1.3.1.tar prom/node-exporter:v1.3.1

docker save -o kubesphere_notification-manager_v2.3.0.tar kubesphere/notification-manager:v2.3.0

docker save -o kubesphere_notification-tenant-sidecar_v3.2.0.tar kubesphere/notification-tenant-sidecar:v3.2.0

docker save -o kubesphere_notification-manager-operator_v2.3.0.tar kubesphere/notification-manager-operator:v2.3.0

docker save -o prom_prometheus_v2.39.1.tar prom/prometheus:v2.39.1

docker save -o kubesphere_prometheus-operator_v0.55.1.tar kubesphere/prometheus-operator:v0.55.1

docker save -o thanosio_thanos_v0.31.0.tar thanosio/thanos:v0.31.0

tmp