暗能星系

zhanglu

[INFO] [12/01/2025 07:33:25.037] [cromwell-system-akka.dispatchers.backend-dispatcher-9548] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-366647aa-46c6-43c2-af7c-3ca2b7aef64b/WorkflowExecutionActor-366647aa-46c6-43c2-af7c-3ca2b7aef64b/366647aa-46c6-43c2-af7c-3ca2b7aef64b-EngineJobExecutionActor-meta_workflow.classbar:NA:1/366647aa-46c6-43c2-af7c-3ca2b7aef64b-BackendJobExecutionActor-meta_workflow.classbar:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(366647aa)meta_workflow.classbar:NA:1]: Job task-ac5a14a7 is complete
[INFO] [12/01/2025 07:33:25.037] [cromwell-system-akka.dispatchers.backend-dispatcher-9526] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-366647aa-46c6-43c2-af7c-3ca2b7aef64b/WorkflowExecutionActor-366647aa-46c6-43c2-af7c-3ca2b7aef64b/366647aa-46c6-43c2-af7c-3ca2b7aef64b-EngineJobExecutionActor-meta_workflow.classbar:NA:1/366647aa-46c6-43c2-af7c-3ca2b7aef64b-BackendJobExecutionActor-meta_workflow.classbar:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(366647aa)meta_workflow.classbar:NA:1]: Status change from Running to Complete

zhanglu

http://192.168.30.202:30090/ga4gh/tes/v1/tasks/task-bc29b16b

{
"id" : "task-bc29b16b",
"state" : "COMPLETE"
}

zhanglu

import requests
import json

def check_task_status(api_url: str) -> None:
"""
调用指定API接口，判断任务状态是否为COMPLETE
:param api_url: 任务状态查询接口URL
"""
try:
# 发送GET请求（添加超时防止无限等待）
response = requests.get(api_url, timeout=10)

    # 检查HTTP响应状态码（200表示请求成功）
    response.raise_for_status()
    
    # 解析JSON响应
    task_data = response.json()
    
    # 提取state字段并判断
    task_state = task_data.get("state", "")  # 无state字段时返回空字符串
    if task_state == "COMPLETE":
        print(f"✅ 任务状态为: {task_state} - 任务已完成！")
    else:
        print(f"❌ 任务状态为: {task_state} - 未完成（目标状态：COMPLETE）")
        
except requests.exceptions.ConnectionError:
    print("❌ 错误：无法连接到接口，请检查URL或网络连接")
except requests.exceptions.Timeout:
    print("❌ 错误：接口请求超时")
except requests.exceptions.HTTPError as e:
    print(f"❌ 错误：HTTP请求失败（状态码：{e.response.status_code}）")
except json.JSONDecodeError:
    print("❌ 错误：接口返回数据不是合法JSON格式")
except Exception as e:
    print(f"❌ 未知错误：{str(e)}")

if name == "main":
# 目标API接口URL
TASK_API_URL = "http://192.168.30.202:30090/ga4gh/tes/v1/tasks/task-bc29b16b"
# 执行状态检查
check_task_status(TASK_API_URL)

import time
if name == "main":
TASK_API_URL = "http://192.168.30.202:30090/ga4gh/tes/v1/tasks/task-bc29b16b"
while True:
check_task_status(TASK_API_URL)
if "COMPLETE" in open("运行日志.txt", "r").read(): # 或直接在函数内判断后退出
break
time.sleep(10) # 每隔10秒查询一次

zhanglu

task-ffe29857 1/1 10s 20d
task-ffe29857-ex-00 1/1 4s 20d
task-ffe41a1d 1/1 101s 21h
task-ffe41a1d-ex-00 1/1 91s 21h
task-ffe44ef0 1/1 2m37s 13d
task-ffe44ef0-ex-00 1/1 2m24s 13d
task-ffe507eb 1/1 74s 5d6h
task-ffe507eb-ex-00 1/1 68s 5d6h
task-ffe54b1c 1/1 101s 4d10h
task-ffe54b1c-ex-00 1/1 88s 4d10h
task-ffea0d86 1/1 28s 11d
task-ffea0d86-ex-00 1/1 23s 11d

zhanglu

#!/bin/bash
set -euo pipefail

定义常量

保留的Job名称（仅示例，脚本会自动匹配成对的Job）

TARGET_PREFIX="task-"

超过7天（转换为秒，7243600=604800秒）

EXPIRE_SECONDS=$((1243600))

当前时间戳（秒）

NOW_TIMESTAMP=$(date +%s)

临时文件存储符合条件的Job列表

TMP_JOB_LIST=$(mktemp)

第一步：获取所有符合命名规则的Completed Job，筛选出超过7天的

echo "===== 开始筛选超过7天且完成的Job ====="
kubectl get jobs --no-headers -o custom-columns="NAME:.metadata.name,COMPLETION_TIME:.status.completionTime,SUCCEEDED:.status.succeeded" |
grep "^${TARGET_PREFIX}" |
while read -r name completion_time succeeded; do
# 跳过未完成的Job（succeeded != 1）
if [ "${succeeded}" != "1" ]; then
continue
fi

# 将completionTime转换为时间戳（处理K8s的时间格式，如 2025-11-25T02:21:10Z）
if ! comp_timestamp=$(date -d "${completion_time}" +%s 2>/dev/null); then
    echo "警告：无法解析Job ${name}的完成时间 ${completion_time}，跳过"
    continue
fi

# 计算已过去的秒数
elapsed_seconds=$((NOW_TIMESTAMP - comp_timestamp))

# 筛选超过7天的Job
if [ ${elapsed_seconds} -gt ${EXPIRE_SECONDS} ]; then
    echo "${name}" >> "${TMP_JOB_LIST}"
    echo "符合条件：${name}（完成时间：${completion_time}，已过去：${elapsed_seconds}秒）"
fi

done

第二步：匹配成对的Job（如 task-ffe54b1c 和 task-ffe54b1c-ex-00），并删除

echo -e "\n===== 开始匹配成对Job并删除 ====="

读取临时文件中的Job列表，按前缀分组

sort "${TMP_JOB_LIST}" | while read -r job_name; do
# 提取核心标识（如从 task-ffe54b1c-ex-00 提取 task-ffe54b1c）
core_id=""
if [[ "${job_name}" =~ ^(${TARGET_PREFIX}[a-f0-9]+)-ex-00$ ]]; then
# 匹配后缀为 -ex-00 的Job，提取核心ID
core_id="${BASH_REMATCH[1]}"
elif [[ "${job_name}" =~ ^${TARGET_PREFIX}[a-f0-9]+$ ]]; then
# 匹配主Job，作为核心ID
core_id="${job_name}"
else
# 不符合命名规则，跳过
continue
fi

# 检查成对的两个Job是否都存在于临时列表中
main_job="${core_id}"
ex_job="${core_id}-ex-00"
if grep -q "^${main_job}$" "${TMP_JOB_LIST}" && grep -q "^${ex_job}$" "${TMP_JOB_LIST}"; then
    # 避免重复删除（删除后从临时文件中移除）
    if ! grep -q "^${main_job}-DELETED$" "${TMP_JOB_LIST}"; then
        echo "删除成对Job：${main_job} 和 ${ex_job}"
        # 执行删除（可先注释kubectl delete，用echo测试）
        kubectl delete job "${main_job}" "${ex_job}"
        # 标记为已删除，避免重复处理
        echo "${main_job}-DELETED" >> "${TMP_JOB_LIST}"
    fi
fi

done

清理临时文件

rm -f "${TMP_JOB_LIST}"

echo -e "\n===== 清理完成 ====="

zhanglu

kubectl delete job task-00160b42
kubectl delete job task-00160b42-ex-00
kubectl delete job task-02fec302
kubectl delete job task-02fec302-ex-00
kubectl delete job task-151cfb1c
kubectl delete job task-151cfb1c-ex-00
kubectl delete job task-188ef6c4
kubectl delete job task-188ef6c4-ex-00
kubectl delete job task-1e22e411
kubectl delete job task-1e22e411-ex-00
kubectl delete job task-20c75c89
kubectl delete job task-20c75c89-ex-00
kubectl delete job task-220bda65
kubectl delete job task-220bda65-ex-00
kubectl delete job task-32db75e1
kubectl delete job task-32db75e1-ex-00
kubectl delete job task-33c0b837
kubectl delete job task-43a99b9a
kubectl delete job task-43a99b9a-ex-00
kubectl delete job task-44aaad8d
kubectl delete job task-44aaad8d-ex-00
kubectl delete job task-4507b81b
kubectl delete job task-4507b81b-ex-00
kubectl delete job task-4ee2fe80
kubectl delete job task-4ee2fe80-ex-00
kubectl delete job task-54f83094
kubectl delete job task-54f83094-ex-00
kubectl delete job task-560205b1
kubectl delete job task-560205b1-ex-00
kubectl delete job task-692bec8a
kubectl delete job task-692bec8a-ex-00
kubectl delete job task-75666b17
kubectl delete job task-75666b17-ex-00
kubectl delete job task-76664940
kubectl delete job task-76664940-ex-00
kubectl delete job task-7974c4cd
kubectl delete job task-7974c4cd-ex-00
kubectl delete job task-7fdfea77
kubectl delete job task-7fdfea77-ex-00
kubectl delete job task-818f1261
kubectl delete job task-818f1261-ex-00
kubectl delete job task-87f89502
kubectl delete job task-87f89502-ex-00
kubectl delete job task-90f2da2c
kubectl delete job task-90f2da2c-ex-00
kubectl delete job task-9d096686
kubectl delete job task-9d096686-ex-00
kubectl delete job task-a7177086
kubectl delete job task-a7177086-ex-00
kubectl delete job task-acc6d856
kubectl delete job task-acc6d856-ex-00
kubectl delete job task-b57e8cbf
kubectl delete job task-b57e8cbf-ex-00
kubectl delete job task-be25389b
kubectl delete job task-be25389b-ex-00
kubectl delete job task-c1c62c24
kubectl delete job task-c1c62c24-ex-00
kubectl delete job task-c64f0b3f
kubectl delete job task-c64f0b3f-ex-00
kubectl delete job task-c64f0b3f
kubectl delete job task-c64f0b3f-ex-00
kubectl delete job task-fe31ccd6
kubectl delete job task-fe31ccd6-ex-00
kubectl delete job task-d0e5ad40
kubectl delete job task-d0e5ad40-ex-00
kubectl delete job task-e9e5a90e
kubectl delete job task-e9e5a90e-ex-00
kubectl delete job task-03e492bb
kubectl delete job task-07508c54
kubectl delete job task-077b4e0a
kubectl delete job task-08a7f0ec
kubectl delete job task-0d350181
kubectl delete job task-1697d3f9
kubectl delete job task-171745c7
kubectl delete job task-18066ac4
kubectl delete job task-190ceba4
kubectl delete job task-1c70f4f3
kubectl delete job task-22b8db7c
kubectl delete job task-262531d1
kubectl delete job task-286ce5f6
kubectl delete job task-2aef062e
kubectl delete job task-34547fae
kubectl delete job task-34fead60
kubectl delete job task-39edd1db
kubectl delete job task-3bdcebb5
kubectl delete job task-413e744c
kubectl delete job task-41c406d9
kubectl delete job task-423e2827
kubectl delete job task-45346a18
kubectl delete job task-46725da3
kubectl delete job task-468aa833
kubectl delete job task-472d9cbf
kubectl delete job task-4c2ffa71
kubectl delete job task-4d2b160e
kubectl delete job task-5058d5cd
kubectl delete job task-53a364df
kubectl delete job task-571aaf94
kubectl delete job task-571bdbe2
kubectl delete job task-57950661
kubectl delete job task-57d9f5ca
kubectl delete job task-5b8bcf07
kubectl delete job task-5f4e799a
kubectl delete job task-68383b65
kubectl delete job task-6a3de99e
kubectl delete job task-6d047488
kubectl delete job task-6da3c25b
kubectl delete job task-6dd8faa6
kubectl delete job task-7156b789
kubectl delete job task-7214b10b
kubectl delete job task-72ba3f8d
kubectl delete job task-76e39fd4
kubectl delete job task-76fb2c78
kubectl delete job task-7877306d
kubectl delete job task-792d60c4
kubectl delete job task-7eae628a
kubectl delete job task-89ac7588
kubectl delete job task-90dc4f05
kubectl delete job task-930cea43
kubectl delete job task-9532cdf3
kubectl delete job task-95e2ea38
kubectl delete job task-a336f510
kubectl delete job task-a934a8e0
kubectl delete job task-ac8b8e39
kubectl delete job task-b12b1d17
kubectl delete job task-b3bc0425
kubectl delete job task-b5007593
kubectl delete job task-b5e8d1ca
kubectl delete job task-b5e8d1ca-ex-00
kubectl delete job task-b640d4e9
kubectl delete job task-b7f14ec9
kubectl delete job task-b86ed9be
kubectl delete job task-bb3c50e1
kubectl delete job task-bb9f9402
kubectl delete job task-bd2681da
kubectl delete job task-c35c5f7a
kubectl delete job task-c58f993b
kubectl delete job task-c833f780
kubectl delete job task-c83b4fda
kubectl delete job task-c8df7262
kubectl delete job task-c8f3fced
kubectl delete job task-c927bd3e
kubectl delete job task-caf7815a
kubectl delete job task-cbc04501
kubectl delete job task-ccc023b9
kubectl delete job task-d150ef6f
kubectl delete job task-d36665f7
kubectl delete job task-d455a2f4
kubectl delete job task-d4d5e409
kubectl delete job task-da04dc0d
kubectl delete job task-dcce4ecf
kubectl delete job task-dd3b1ba4
kubectl delete job task-df6d5230
kubectl delete job task-e441f7b9
kubectl delete job task-e70a4cd6
kubectl delete job task-e88fca64
kubectl delete job task-e968c11c
kubectl delete job task-ec061560
kubectl delete job task-ec47dc5f
kubectl delete job task-eee5ff5a
kubectl delete job task-ef9f195f
kubectl delete job task-f5cd2e6e
kubectl delete job task-fb8836da
kubectl delete job task-fdaeb6eb
kubectl delete job task-fe94a44b
kubectl delete job task-ffd556e2

zhanglu

[INFO] [12/18/2025 06:45:43.162] [cromwell-system-akka.dispatchers.api-dispatcher-139] [akka://cromwell-system/user/cromwell-service/WorkflowStoreActor/WorkflowStoreSubmitActor] Unspecified type (Unspecified version) workflow 5b890685-2085-4e03-8465-c317b1e1f23d submitted

[INFO] [12/18/2025 06:47:53.027] [cromwell-system-akka.dispatchers.engine-dispatcher-57] [akka://cromwell-system/user/cromwell-service/WorkflowStoreActor/WorkflowStoreEngineActor] 1 new workflows fetched by cromid-d6f0ebc: 5b890685-2085-4e03-8465-c317b1e1f23d
[INFO] [12/18/2025 06:47:53.027] [cromwell-system-akka.dispatchers.engine-dispatcher-57] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Starting workflow UUID(5b890685-2085-4e03-8465-c317b1e1f23d)
[INFO] [12/18/2025 06:47:53.027] [cromwell-system-akka.dispatchers.engine-dispatcher-57] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Successfully started WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d
[INFO] [12/18/2025 06:47:53.029] [cromwell-system-akka.dispatchers.engine-dispatcher-55] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/MaterializeWorkflowDescriptorActor] MaterializeWorkflowDescriptorActor [UUID(5b890685)]: Parsing workflow as WDL draft-2
[INFO] [12/18/2025 06:47:53.031] [cromwell-system-akka.dispatchers.engine-dispatcher-55] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/MaterializeWorkflowDescriptorActor] MaterializeWorkflowDescriptorActor [UUID(5b890685)]: Call-to-Backend assignments: msConvert.extractTICdata -> TESK
[INFO] [12/18/2025 06:47:55.101] [cromwell-system-akka.dispatchers.engine-dispatcher-82] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d] WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d [UUID(5b890685)]: Starting msConvert.extractTICdata
[WARN] [12/18/2025 06:48:04.714] [cromwell-system-akka.dispatchers.engine-dispatcher-55] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/BackendPreparationActor_for_5b890685:msConvert.extractTICdata1] BackendPreparationActor_for_5b890685:msConvert.extractTICdata1 [UUID(5b890685)]: Docker lookup failed
[INFO] [12/18/2025 06:48:05.087] [cromwell-system-akka.dispatchers.backend-dispatcher-1210] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: echo "192.168.30.202:23099/target_meta/target_meta_dev:v3.3.7"
[INFO] [12/18/2025 06:48:05.087] [cromwell-system-akka.dispatchers.backend-dispatcher-1210] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: Calculated TES outputs (found 4):
Output(Some(rc),Some(msConvert.extractTICdata.rc),Some(/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/rc),/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/rc,Some(FILE))
Output(Some(stdout),Some(msConvert.extractTICdata.stdout),Some(/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/stdout),/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/stdout,Some(FILE))
Output(Some(stderr),Some(msConvert.extractTICdata.stderr),Some(/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/stderr),/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/stderr,Some(FILE))
Output(Some(commandScript),Some(msConvert.extractTICdata.commandScript),Some(/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/script),/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/script,Some(FILE))
[INFO] [12/18/2025 06:48:05.087] [cromwell-system-akka.dispatchers.backend-dispatcher-1210] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: Calculated TES inputs (found 1):
cromwell.backend.impl.tes.Input(Some(commandScript),Some(msConvert.extractTICdata.commandScript),Some(/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/script),/cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions/msConvert/5b890685-2085-4e03-8465-c317b1e1f23d/call-extractTICdata/execution/script,Some(FILE),None)
[INFO] [12/18/2025 06:48:05.087] [cromwell-system-akka.dispatchers.backend-dispatcher-1210] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: echo "192.168.30.202:23099/target_meta/target_meta_dev:v3.3.7"
[INFO] [12/18/2025 06:48:09.544] [cromwell-system-akka.dispatchers.backend-dispatcher-1175] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: job id: task-3c5d0482
[INFO] [12/18/2025 06:48:09.681] [cromwell-system-akka.dispatchers.backend-dispatcher-1138] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: Status change from - to Running
[INFO] [12/18/2025 06:52:14.785] [cromwell-system-akka.dispatchers.backend-dispatcher-984] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: Job task-3c5d0482 is complete
[INFO] [12/18/2025 06:52:14.785] [cromwell-system-akka.dispatchers.backend-dispatcher-1141] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d/5b890685-2085-4e03-8465-c317b1e1f23d-EngineJobExecutionActor-msConvert.extractTICdata:NA:1/5b890685-2085-4e03-8465-c317b1e1f23d-BackendJobExecutionActor-msConvert.extractTICdata:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(5b890685)msConvert.extractTICdata:NA:1]: Status change from Running to Complete
[INFO] [12/18/2025 06:52:16.222] [cromwell-system-akka.dispatchers.engine-dispatcher-112] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-5b890685-2085-4e03-8465-c317b1e1f23d/WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d] WorkflowExecutionActor-5b890685-2085-4e03-8465-c317b1e1f23d [UUID(5b890685)]: Workflow msConvert complete. Final Outputs:
[INFO] [12/18/2025 06:52:19.549] [cromwell-system-akka.dispatchers.engine-dispatcher-81] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Workflow actor for 5b890685-2085-4e03-8465-c317b1e1f23d completed with status 'Succeeded'. The workflow will be removed from the workflow store.

zhanglu

[INFO] [12/18/2025 07:05:32.113] [cromwell-system-akka.dispatchers.backend-dispatcher-1244] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-9c9d7421-4218-4a3f-9f46-87faa5edc47e/WorkflowExecutionActor-9c9d7421-4218-4a3f-9f46-87faa5edc47e/9c9d7421-4218-4a3f-9f46-87faa5edc47e-EngineJobExecutionActor-monoMeth.jsonFile:NA:1/9c9d7421-4218-4a3f-9f46-87faa5edc47e-BackendJobExecutionActor-monoMeth.jsonFile:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(9c9d7421)monoMeth.jsonFile:NA:1]: Job task-9a3b0fd9 is complete
[INFO] [12/18/2025 07:05:32.113] [cromwell-system-akka.dispatchers.backend-dispatcher-1255] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-9c9d7421-4218-4a3f-9f46-87faa5edc47e/WorkflowExecutionActor-9c9d7421-4218-4a3f-9f46-87faa5edc47e/9c9d7421-4218-4a3f-9f46-87faa5edc47e-EngineJobExecutionActor-monoMeth.jsonFile:NA:1/9c9d7421-4218-4a3f-9f46-87faa5edc47e-BackendJobExecutionActor-monoMeth.jsonFile:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(9c9d7421)monoMeth.jsonFile:NA:1]: Status change from Running to Complete

zhanglu

[INFO] [12/18/2025 07:26:16.905] [cromwell-system-akka.dispatchers.api-dispatcher-1503] [akka://cromwell-system/user/cromwell-service/WorkflowStoreActor/WorkflowStoreSubmitActor] Unspecified type (Unspecified version) workflow b3c8ad44-dc3b-4e87-9fe9-97c0db47031b submitted
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowStoreActor/WorkflowStoreEngineActor] 1 new workflows fetched by cromid-247fb6c: b3c8ad44-dc3b-4e87-9fe9-97c0db47031b
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Starting workflow UUID(b3c8ad44-dc3b-4e87-9fe9-97c0db47031b)
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Successfully started WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] Retrieved 1 workflows from the WorkflowStoreActor
[INFO] [12/18/2025 07:26:31.293] [cromwell-system-akka.dispatchers.engine-dispatcher-74] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/MaterializeWorkflowDescriptorActor] MaterializeWorkflowDescriptorActor [UUID(b3c8ad44)]: Parsing workflow as WDL draft-2
[INFO] [12/18/2025 07:26:31.298] [cromwell-system-akka.dispatchers.engine-dispatcher-74] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/MaterializeWorkflowDescriptorActor] MaterializeWorkflowDescriptorActor [UUID(b3c8ad44)]: Call-to-Backend assignments: meta_workflow.Identify_analy -> TESK
[INFO] [12/18/2025 07:26:33.508] [cromwell-system-akka.dispatchers.engine-dispatcher-74] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b] WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b [UUID(b3c8ad44)]: Starting meta_workflow.Identify_analy
[INFO] [12/18/2025 07:26:41.267] [cromwell-system-akka.dispatchers.engine-dispatcher-55] [akka://cromwell-system/user/cromwell-service/JobExecutionTokenDispenser] Assigned new job execution tokens to the following groups: b3c8ad44: 1
[WARN] [12/18/2025 07:26:41.289] [cromwell-system-akka.dispatchers.engine-dispatcher-66] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/BackendPreparationActor_for_b3c8ad44:meta_workflow.Identify_analy1] BackendPreparationActor_for_b3c8ad44:meta_workflow.Identify_analy1 [UUID(b3c8ad44)]: Docker lookup failed
java.lang.Exception: Failed to get docker hash for dockerhub.genostack.com:8090/library/ubuntu:18.04 Error running: docker images --digests --format {{printf "%s\t%s\t%s" .Repository .Tag .Digest}}
Exit code: 1
Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?

at cromwell.engine.workflow.WorkflowDockerLookupActor.cromwell$engine$workflow$WorkflowDockerLookupActor$$handleLookupFailure(WorkflowDockerLookupActor.scala:279)
at cromwell.engine.workflow.WorkflowDockerLookupActor$$anonfun$3.applyOrElse(WorkflowDockerLookupActor.scala:93)
at cromwell.engine.workflow.WorkflowDockerLookupActor$$anonfun$3.applyOrElse(WorkflowDockerLookupActor.scala:78)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
at akka.actor.FSM.processEvent(FSM.scala:707)
at akka.actor.FSM.processEvent$(FSM.scala:704)
at cromwell.engine.workflow.WorkflowDockerLookupActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowDockerLookupActor.scala:45)
at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
at cromwell.engine.workflow.WorkflowDockerLookupActor.processEvent(WorkflowDockerLookupActor.scala:45)
at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
at cromwell.docker.DockerClientHelper$$anonfun$dockerResponseReceive$1.applyOrElse(DockerClientHelper.scala:16)
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:269)
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:270)
at akka.actor.Actor.aroundReceive(Actor.scala:539)
at akka.actor.Actor.aroundReceive$(Actor.scala:537)
at cromwell.engine.workflow.WorkflowDockerLookupActor.aroundReceive(WorkflowDockerLookupActor.scala:45)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
at akka.actor.ActorCell.invoke(ActorCell.scala:583)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
at akka.dispatch.Mailbox.run(Mailbox.scala:229)
at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

[INFO] [12/18/2025 07:26:41.631] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: echo 'DATABASE' > result.txt
[INFO] [12/18/2025 07:26:41.632] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Calculated TES outputs (found 5):
Output(Some(rc),Some(meta_workflow.Identify_analy.rc),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/rc),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/rc,Some(FILE))
Output(Some(stdout),Some(meta_workflow.Identify_analy.stdout),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stdout),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stdout,Some(FILE))
Output(Some(stderr),Some(meta_workflow.Identify_analy.stderr),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stderr),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stderr,Some(FILE))
Output(Some(commandScript),Some(meta_workflow.Identify_analy.commandScript),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script,Some(FILE))
Output(Some(meta_workflow.Identify_analy.output.0),Some(meta_workflow.Identify_analy.output.0),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/result.txt),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/result.txt,Some(FILE))

[INFO] [12/18/2025 07:26:41.632] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Calculated TES inputs (found 1):
cromwell.backend.impl.tes.Input(Some(commandScript),Some(meta_workflow.Identify_analy.commandScript),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script,Some(FILE),None)

[INFO] [12/18/2025 07:26:41.632] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: echo 'DATABASE' > result.txt
[INFO] [12/18/2025 07:26:46.175] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: job id: task-710bdfb8
[INFO] [12/18/2025 07:26:47.921] [cromwell-system-akka.dispatchers.backend-dispatcher-1436] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Status change from - to Running
[INFO] [12/18/2025 07:32:14.919] [cromwell-system-akka.dispatchers.backend-dispatcher-1436] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Job task-710bdfb8 is complete
[INFO] [12/18/2025 07:32:14.919] [cromwell-system-akka.dispatchers.backend-dispatcher-1436] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Status change from Running to Complete
[INFO] [12/18/2025 07:32:16.299] [cromwell-system-akka.dispatchers.engine-dispatcher-47] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b] WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b [UUID(b3c8ad44)]: Workflow meta_workflow complete. Final Outputs:
{
"meta_workflow.Identify_analy.out": "/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/result.txt"
}
[INFO] [12/18/2025 07:32:21.193] [cromwell-system-akka.dispatchers.engine-dispatcher-82] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Workflow actor for b3c8ad44-dc3b-4e87-9fe9-97c0db47031b completed with status 'Succeeded'. The workflow will be removed from the workflow store.

zhanglu

[INFO] [12/18/2025 07:26:16.905] [cromwell-system-akka.dispatchers.api-dispatcher-1503] [akka://cromwell-system/user/cromwell-service/WorkflowStoreActor/WorkflowStoreSubmitActor] Unspecified type (Unspecified version) workflow b3c8ad44-dc3b-4e87-9fe9-97c0db47031b submitted
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowStoreActor/WorkflowStoreEngineActor] 1 new workflows fetched by cromid-247fb6c: b3c8ad44-dc3b-4e87-9fe9-97c0db47031b
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Starting workflow UUID(b3c8ad44-dc3b-4e87-9fe9-97c0db47031b)
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Successfully started WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b
[INFO] [12/18/2025 07:26:31.291] [cromwell-system-akka.dispatchers.engine-dispatcher-68] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] Retrieved 1 workflows from the WorkflowStoreActor
[INFO] [12/18/2025 07:26:31.293] [cromwell-system-akka.dispatchers.engine-dispatcher-74] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/MaterializeWorkflowDescriptorActor] MaterializeWorkflowDescriptorActor [UUID(b3c8ad44)]: Parsing workflow as WDL draft-2
[INFO] [12/18/2025 07:26:31.298] [cromwell-system-akka.dispatchers.engine-dispatcher-74] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/MaterializeWorkflowDescriptorActor] MaterializeWorkflowDescriptorActor [UUID(b3c8ad44)]: Call-to-Backend assignments: meta_workflow.Identify_analy -> TESK
[INFO] [12/18/2025 07:26:33.508] [cromwell-system-akka.dispatchers.engine-dispatcher-74] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b] WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b [UUID(b3c8ad44)]: Starting meta_workflow.Identify_analy
[INFO] [12/18/2025 07:26:41.267] [cromwell-system-akka.dispatchers.engine-dispatcher-55] [akka://cromwell-system/user/cromwell-service/JobExecutionTokenDispenser] Assigned new job execution tokens to the following groups: b3c8ad44: 1
[WARN] [12/18/2025 07:26:41.289] [cromwell-system-akka.dispatchers.engine-dispatcher-66] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/BackendPreparationActor_for_b3c8ad44:meta_workflow.Identify_analy1] BackendPreparationActor_for_b3c8ad44:meta_workflow.Identify_analy1 [UUID(b3c8ad44)]: Docker lookup failed
java.lang.Exception: Failed to get docker hash for dockerhub.genostack.com:8090/library/ubuntu:18.04 Error running: docker images --digests --format {{printf "%s\t%s\t%s" .Repository .Tag .Digest}}
Exit code: 1
Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?

at cromwell.engine.workflow.WorkflowDockerLookupActor.cromwell$engine$workflow$WorkflowDockerLookupActor$$handleLookupFailure(WorkflowDockerLookupActor.scala:279)
at cromwell.engine.workflow.WorkflowDockerLookupActor$$anonfun$3.applyOrElse(WorkflowDockerLookupActor.scala:93)
at cromwell.engine.workflow.WorkflowDockerLookupActor$$anonfun$3.applyOrElse(WorkflowDockerLookupActor.scala:78)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
at akka.actor.FSM.processEvent(FSM.scala:707)
at akka.actor.FSM.processEvent$(FSM.scala:704)
at cromwell.engine.workflow.WorkflowDockerLookupActor.akka$actor$LoggingFSM$$super$processEvent(WorkflowDockerLookupActor.scala:45)
at akka.actor.LoggingFSM.processEvent(FSM.scala:847)
at akka.actor.LoggingFSM.processEvent$(FSM.scala:829)
at cromwell.engine.workflow.WorkflowDockerLookupActor.processEvent(WorkflowDockerLookupActor.scala:45)
at akka.actor.FSM.akka$actor$FSM$$processMsg(FSM.scala:701)
at akka.actor.FSM$$anonfun$receive$1.applyOrElse(FSM.scala:695)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:35)
at cromwell.docker.DockerClientHelper$$anonfun$dockerResponseReceive$1.applyOrElse(DockerClientHelper.scala:16)
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:269)
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:270)
at akka.actor.Actor.aroundReceive(Actor.scala:539)
at akka.actor.Actor.aroundReceive$(Actor.scala:537)
at cromwell.engine.workflow.WorkflowDockerLookupActor.aroundReceive(WorkflowDockerLookupActor.scala:45)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:614)
at akka.actor.ActorCell.invoke(ActorCell.scala:583)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)
at akka.dispatch.Mailbox.run(Mailbox.scala:229)
at akka.dispatch.Mailbox.exec(Mailbox.scala:241)
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

[INFO] [12/18/2025 07:26:41.631] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: echo 'DATABASE' > result.txt
[INFO] [12/18/2025 07:26:41.632] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Calculated TES outputs (found 5):
Output(Some(rc),Some(meta_workflow.Identify_analy.rc),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/rc),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/rc,Some(FILE))
Output(Some(stdout),Some(meta_workflow.Identify_analy.stdout),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stdout),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stdout,Some(FILE))
Output(Some(stderr),Some(meta_workflow.Identify_analy.stderr),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stderr),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/stderr,Some(FILE))
Output(Some(commandScript),Some(meta_workflow.Identify_analy.commandScript),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script,Some(FILE))
Output(Some(meta_workflow.Identify_analy.output.0),Some(meta_workflow.Identify_analy.output.0),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/result.txt),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/result.txt,Some(FILE))

[INFO] [12/18/2025 07:26:41.632] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Calculated TES inputs (found 1):
cromwell.backend.impl.tes.Input(Some(commandScript),Some(meta_workflow.Identify_analy.commandScript),Some(/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script),/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/script,Some(FILE),None)

[INFO] [12/18/2025 07:26:41.632] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: echo 'DATABASE' > result.txt
[INFO] [12/18/2025 07:26:46.175] [cromwell-system-akka.dispatchers.backend-dispatcher-1467] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: job id: task-710bdfb8
[INFO] [12/18/2025 07:26:47.921] [cromwell-system-akka.dispatchers.backend-dispatcher-1436] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Status change from - to Running
[INFO] [12/18/2025 07:32:14.919] [cromwell-system-akka.dispatchers.backend-dispatcher-1436] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Job task-710bdfb8 is complete
[INFO] [12/18/2025 07:32:14.919] [cromwell-system-akka.dispatchers.backend-dispatcher-1436] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-EngineJobExecutionActor-meta_workflow.Identify_analy:NA:1/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b-BackendJobExecutionActor-meta_workflow.Identify_analy:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(b3c8ad44)meta_workflow.Identify_analy:NA:1]: Status change from Running to Complete
[INFO] [12/18/2025 07:32:16.299] [cromwell-system-akka.dispatchers.engine-dispatcher-47] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b] WorkflowExecutionActor-b3c8ad44-dc3b-4e87-9fe9-97c0db47031b [UUID(b3c8ad44)]: Workflow meta_workflow complete. Final Outputs:
{
"meta_workflow.Identify_analy.out": "/cephfs_data/genostack_v3_dev/genostack_cromwell/cromwell-executions/meta_workflow/b3c8ad44-dc3b-4e87-9fe9-97c0db47031b/call-Identify_analy/execution/result.txt"
}
[INFO] [12/18/2025 07:32:21.193] [cromwell-system-akka.dispatchers.engine-dispatcher-82] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Workflow actor for b3c8ad44-dc3b-4e87-9fe9-97c0db47031b completed with status 'Succeeded'. The workflow will be removed from the workflow store.

zhanglu

[INFO] [12/18/2025 07:55:14.436] [cromwell-system-akka.dispatchers.backend-dispatcher-1296] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-6de70bcd-083a-4fd7-9198-9cbb2005dbe5/WorkflowExecutionActor-6de70bcd-083a-4fd7-9198-9cbb2005dbe5/6de70bcd-083a-4fd7-9198-9cbb2005dbe5-EngineJobExecutionActor-untarget_workflow.chemrich:NA:1/6de70bcd-083a-4fd7-9198-9cbb2005dbe5-BackendJobExecutionActor-untarget_workflow.chemrich:NA:1/TesAsyncBackendJobExecutionActor] TesAsyncBackendJobExecutionActor [UUID(6de70bcd)untarget_workflow.chemrich:NA:1]: Status change from Running to Complete

zhanglu

2025-12-19 14:46:41.752 [container-3592] INFO c.a.c.redis.MessagePubListener - 消费者消费: {"workflowId":"6d8348ea-1893-4b86-b9f7-d2616826f82c[INFO] [12/19/2025 06:45:22.157] [cromwell-system-akka.dispatchers.engine-dispatcher-83] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/b16ea7b9-0d2d-4f29-9ac0-cc8a0a6722bf/WorkflowExecutionActor-b16ea7b9-0d2d-4f29-9ac0-cc8a0a6722bf] WorkflowExecutionActor-b16ea7b9-0d2d-4f29-9ac0-cc8a0a6722bf [UUID(b16ea7b9)]: Starting amplicon_meta.W01_cleandata", "taskName":"", "status":"start", "type":"workflow", "time":"2025-12-19 14:46:36"}

[INFO] [12/19/2025 06:45:25.109] [cromwell-system-akka.dispatchers.engine-dispatcher-97] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor: Successfully started WorkflowActor-6d8348ea-1893-4b86-b9f7-d2616826f82c[INFO] [12/19/2025 06:45:22.157] [cromwell-system-akka.dispatchers.engine-dispatcher-83] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-b16ea7b9-0d2d-4f29-9ac0-cc8a0a6722bf/WorkflowExecutionActor-b16ea7b9-0d2d-4f29-9ac0-cc8a0a6722bf] WorkflowExecutionActor-b16ea7b9-0d2d-4f29-9ac0-cc8a0a6722bf [UUID(b16ea7b9)]: Starting amplicon_meta.W01_cleandata

zhanglu

5-12-23 06:22:21.512 UTC [26] LOG: database system was interrupted; last known up at 2025-12-23 06:21:21 UTC
2025-12-23 06:22:21.950 UTC [26] LOG: database system was not properly shut down; automatic recovery in progress
2025-12-23 06:22:21.954 UTC [26] LOG: redo starts at 18/9B999808
2025-12-23 06:22:21.955 UTC [26] LOG: invalid record length at 18/9BA06F18: wanted 24, got 0
2025-12-23 06:22:21.955 UTC [26] LOG: redo done at 18/9BA06EF0
2025-12-23 06:22:21.955 UTC [26] LOG: last completed transaction was at log time 2025-12-23 06:22:20.084128+00
2025-12-23 06:22:22.075 UTC [1] LOG: database system is ready to accept connections

zhanglu

cluster:
id: 807d820b-5c5b-451c-9f52-41b93d5d905a
health: HEALTH_ERR
1 MDSs report oversized cache
1 MDSs are read only
2 MDSs behind on trimming
full ratio(s) out of order
Low space hindering backfill (add storage if this doesn't resolve itself): 10 pgs backfill_toofull
94 pgs not deep-scrubbed in time
69 pgs not scrubbed in time

services:
mon: 3 daemons, quorum bm,bo,bp (age 13d)
mgr: b(active, since 2w), standbys: a
mds: 1/1 daemons up, 1 hot standby
osd: 51 osds: 51 up (since 13d), 51 in (since 2w); 67 remapped pgs

data:
volumes: 1/1 healthy
pools: 4 pools, 2097 pgs
objects: 279.98M objects, 306 TiB
usage: 619 TiB used, 82 TiB / 701 TiB avail
pgs: 3748313/601598194 objects misplaced (0.623%)
2024 active+clean
57 active+remapped+backfilling
10 active+remapped+backfill_toofull
6 active+clean+scrubbing+deep

io:
client: 1.2 MiB/s rd, 100 KiB/s wr, 183 op/s rd, 19 op/s wr
recovery: 52 MiB/s, 37 objects/s

zhanglu

cluster:
id: 807d820b-5c5b-451c-9f52-41b93d5d905a
health: HEALTH_ERR
1 MDSs report oversized cache
1 MDSs are read only
2 MDSs behind on trimming
full ratio(s) out of order
Low space hindering backfill (add storage if this doesn't resolve itself): 10 pgs backfill_toofull
94 pgs not deep-scrubbed in time
69 pgs not scrubbed in time

services:
mon: 3 daemons, quorum bm,bo,bp (age 13d)
mgr: b(active, since 2w), standbys: a
mds: 1/1 daemons up, 1 hot standby
osd: 51 osds: 51 up (since 13d), 51 in (since 2w); 67 remapped pgs

data:
volumes: 1/1 healthy
pools: 4 pools, 2097 pgs
objects: 279.98M objects, 306 TiB
usage: 619 TiB used, 82 TiB / 701 TiB avail
pgs: 3748313/601598194 objects misplaced (0.623%)
2024 active+clean
57 active+remapped+backfilling
10 active+remapped+backfill_toofull
6 active+clean+scrubbing+deep

io:
client: 1.2 MiB/s rd, 100 KiB/s wr, 183 op/s rd, 19 op/s wr
recovery: 52 MiB/s, 37 objects/s

zhanglu

Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-attacher:v4.0.0
Image:         registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0
Image:         registry.k8s.io/sig-storage/csi-resizer:v1.6.0
Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-attacher:v4.0.0
Image:         registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0
Image:         registry.k8s.io/sig-storage/csi-resizer:v1.6.0
Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
Image:         registry.k8s.io/sig-storage/csi-resizer:v1.6.0
Image:         registry.k8s.io/sig-storage/csi-attacher:v4.0.0
Image:         registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
Image:         registry.k8s.io/sig-storage/csi-resizer:v1.6.0
Image:         registry.k8s.io/sig-storage/csi-attacher:v4.0.0
Image:         registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         rook/ceph:v1.10.8
Image:         quay.io/cephcsi/cephcsi:v3.7.2
Image:         rook/ceph:v1.10.8
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         rook/ceph:v1.10.8
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         rook/ceph:v1.10.8
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         rook/ceph:v1.10.8
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5
Image:         rook/ceph:v1.10.8
Image:         quay.io/ceph/ceph:v17.2.5
Image:         quay.io/ceph/ceph:v17.2.5

zhanglu

Image:          quay.io/calico/kube-controllers:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:         quay.io/calico/cni:v3.16.5
Image:          quay.io/calico/node:v3.16.5
Image:         k8s.gcr.io/coredns:1.7.0
Image:         k8s.gcr.io/coredns:1.7.0
Image:         k8s.gcr.io/coredns:1.7.0
Image:         k8s.gcr.io/coredns:1.7.0
Image:         k8s.gcr.io/coredns:1.7.0
Image:         k8s.gcr.io/coredns:1.7.0
Image:         k8s.gcr.io/cpa/cluster-proportional-autoscaler-amd64:1.8.3
Image:         k8s.gcr.io/kube-apiserver:v1.19.7
Image:         k8s.gcr.io/kube-apiserver:v1.19.7
Image:         k8s.gcr.io/kube-controller-manager:v1.19.7
Image:         k8s.gcr.io/kube-controller-manager:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-proxy:v1.19.7
Image:         k8s.gcr.io/kube-scheduler:v1.19.7
Image:         k8s.gcr.io/kube-scheduler:v1.19.7
Image:          docker.io/library/nginx:1.19
Image:          docker.io/library/nginx:1.19
Image:          docker.io/library/nginx:1.19
Image:          docker.io/library/nginx:1.19
Image:          docker.io/library/nginx:1.19
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:         k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:          nvcr.io/nvidia/k8s-device-plugin:v0.17.1
Image:         csiplugin/snapshot-controller:v4.0.0

zhanglu

docker save registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 -o csi-node-driver-registrar.tar
docker save quay.io/cephcsi/cephcsi:v3.7.2 -o cephcsi.tgar
docker save registry.k8s.io/sig-storage/csi-attacher:v4.0.0 -o csi-attacher.tar
docker save registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 -o csi-snapshotter.tar
docker save registry.k8s.io/sig-storage/csi-resizer:v1.6.0 -o csi-resizer.tar
docker save registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 -o csi-provisioner.tar
docker save quay.io/ceph/ceph:v17.2.5 -o ceph1.tar
docker save rook/ceph:v1.10.8 -o ceph.tar

docker save quay.io/calico/kube-controllers:v3.16.5 -o kube-controllers.tar
docker save quay.io/calico/cni:v3.16.5 -o cni.tar
docker save quay.io/calico/node:v3.16.5 -o node.tar
docker save k8s.gcr.io/coredns:1.7.0 -o coredns.tar
docker save k8s.gcr.io/cpa/cluster-proportional-autoscaler-amd64:1.8.3 -o cluster-proportional-autoscaler-amd64.tar
docker save k8s.gcr.io/kube-apiserver:v1.19.7 -o apiserver.tar
docker save k8s.gcr.io/kube-controller-manager:v1.19.7 -o manager.tar
docker save k8s.gcr.io/kube-proxy:v1.19.7 -o proxy.tar
docker save k8s.gcr.io/kube-scheduler:v1.19.7 -o scheduler.tar
docker save docker.io/library/nginx:1.19 -o ngins.tar
docker save k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0 -o cache.tar
docker save nvcr.io/nvidia/k8s-device-plugin:v0.17.1 -o plugin.tar
docker save csiplugin/snapshot-controller:v4.0.0 -o controller.tar

zhanglu

docker load < csi-node-driver-registrar.tar
docker load < cephcsi.tgar
docker load <csi-attacher.tar
docker load < csi-snapshotter.tar
docker load < csi-resizer.tar
docker load < csi-provisioner.tar
docker load <-o ceph1.tar
docker load < -o ceph.tar

docker load < kube-controllers.tar
docker load < cni.tar
docker load < node.tar
docker load < coredns.tar
docker load < cluster-proportional-autoscaler-amd64.tar
docker load < -o apiserver.tar
docker load < manager.tar
docker load < proxy.tar
docker load < scheduler.tar
docker load < ngins.tar
docker load < cache.tar
docker load < plugin.tar
docker load <controller.tar

zhanglu

cluster:
id: 807d820b-5c5b-451c-9f52-41b93d5d905a
health: HEALTH_ERR
full ratio(s) out of order
Low space hindering backfill (add storage if this doesn't resolve itself): 5 pgs backfill_toofull
178 pgs not deep-scrubbed in time
117 pgs not scrubbed in time

services:
mon: 3 daemons, quorum bm,bo,bp (age 39h)
mgr: a(active, since 39h), standbys: b
mds: 1/1 daemons up, 1 hot standby
osd: 51 osds: 51 up (since 109m), 51 in (since 109m); 59 remapped pgs

data:
volumes: 1/1 healthy
pools: 4 pools, 2097 pgs
objects: 279.97M objects, 306 TiB
usage: 617 TiB used, 84 TiB / 701 TiB avail
pgs: 5001255/601565087 objects misplaced (0.831%)
2028 active+clean
54 active+remapped+backfilling
10 active+clean+scrubbing+deep
5 active+remapped+backfill_toofull

io:
client: 338 KiB/s rd, 8.8 MiB/s wr, 175 op/s rd, 7 op/s wr
recovery: 65 MiB/s, 49 objects/s

tmp