umount /cephfs_data/genostack_v3/genostack_scriptCode
umount /cephfs_data/genostack_v3/genostack_cromwell/cromwell-executions
umount /cephfs_data/genostack_v3/genostack_php/public_file_data/
umount /cephfs_data/genostack_v3/genostack_php/project_data

NAME    STATUS   ROLES           AGE    VERSION
node1   Ready    control-plane   169d   v1.27.7
node2   Ready    control-plane   169d   v1.27.7
node3   Ready    <none>          169d   v1.27.7

kubectl get namespace

kubectl get pod -n kube-system

calico-node-hgmst                          1/1     Running   7 (65d ago)     168d
calico-node-kmmfq                          1/1     Running   0               25d
calico-node-l52r8                          1/1     Running   1 (167d ago)    168d
coredns-5c469774b8-29czz                   1/1     Running   0               26d
coredns-5c469774b8-4qh9l                   1/1     Running   0               165d
dns-autoscaler-f455cf558-lf5jb             1/1     Running   0               163d
kube-apiserver-node1                       1/1     Running   87 (25d ago)    26d
kube-apiserver-node2                       1/1     Running   226 (65d ago)   169d
kube-controller-manager-node1              1/1     Running   83 (25d ago)    25d
kube-controller-manager-node2              1/1     Running   39 (12d ago)    169d
kube-proxy-25qlx                           1/1     Running   1 (167d ago)    168d
kube-proxy-7kjtt                           1/1     Running   9 (25d ago)     168d
kube-proxy-h5mkk                           1/1     Running   7 (65d ago)     168d
kube-scheduler-node1                       1/1     Running   72 (25d ago)    25d
kube-scheduler-node2                       1/1     Running   37 (12d ago)    169d
metrics-server-66549d4ff5-hs6zn            1/1     Running   0               24d
nginx-proxy-node3                          1/1     Running   183 (65d ago)   169d
nodelocaldns-f4mtg                         1/1     Running   504 (64d ago)   168d
nodelocaldns-n47wf                         1/1     Running   98 (25d ago)    168d
nodelocaldns-xhqj9                         1/1     Running   908 (64d ago)   168d
snapshot-controller-0                      1/1     Running   0               25d

kubectl get pod -n rook-ceph

NAME                                              READY   STATUS      RESTARTS        AGE
csi-cephfsplugin-6t2lg                            2/2     Running     0               164d
csi-cephfsplugin-bcj2v                            2/2     Running     24 (25d ago)    164d
csi-cephfsplugin-provisioner-5b59dbc849-8n879     5/5     Running     47 (21h ago)    163d
csi-cephfsplugin-provisioner-5b59dbc849-vcttq     5/5     Running     4 (7d21h ago)   26d
csi-cephfsplugin-zjbvt                            2/2     Running     12 (65d ago)    164d
csi-rbdplugin-h76b8                               2/2     Running     12 (65d ago)    164d
csi-rbdplugin-n85dj                               2/2     Running     23 (25d ago)    164d
csi-rbdplugin-provisioner-59777f956d-8zt6f        5/5     Running     0               26d
csi-rbdplugin-provisioner-59777f956d-tdz77        5/5     Running     21 (65d ago)    163d
csi-rbdplugin-wr94r                               2/2     Running     0               164d
rook-ceph-crashcollector-node1-6797f99845-6mnxf   1/1     Running     2 (25d ago)     26d
rook-ceph-crashcollector-node2-6599d4856-wzmqp    1/1     Running     0               155d
rook-ceph-crashcollector-node3-7769d57785-wk7t7   1/1     Running     0               56d
rook-ceph-exporter-node1-b5589c46f-nxct7          1/1     Running     3 (25d ago)     26d
rook-ceph-exporter-node2-756fc8668-zv7wg          1/1     Running     0               155d
rook-ceph-exporter-node3-5c5ddc477c-q82rq         1/1     Running     0               56d
rook-ceph-mds-myfs-ec-a-847b49d7fd-njctk          2/2     Running     5 (26d ago)     56d
rook-ceph-mds-myfs-ec-b-f676f7fbd-4r5vg           2/2     Running     82 (26d ago)    155d
rook-ceph-mgr-a-bf8f784f9-7m4cw                   3/3     Running     0               26d
rook-ceph-mgr-b-59f8d89bfb-fkshl                  3/3     Running     0               160d
rook-ceph-mon-c-84f75f85d8-h2mv4                  2/2     Running     0               164d
rook-ceph-mon-f-687bf74f58-chqqj                  2/2     Running     0               64d
rook-ceph-mon-g-55fd64cdc7-kch85                  2/2     Running     4 (25d ago)     25d
rook-ceph-operator-544c5c6477-qkm2q               1/1     Running     0               160d
rook-ceph-osd-0-774465f4c7-sljpn                  2/2     Running     0               164d
rook-ceph-osd-1-795795559b-4h92l                  2/2     Running     0               65d
rook-ceph-osd-2-7f69b64959-pqt65                  2/2     Running     0               164d
rook-ceph-osd-3-5d458b5794-jfczk                  2/2     Running     0               65d
rook-ceph-osd-4-5bbc59d7cd-n5wtn                  2/2     Running     0               164d
rook-ceph-osd-5-894df5d9f-274jw                   2/2     Running     0               65d
rook-ceph-osd-6-77bcc66bb-4gvnn                   2/2     Running     0               65d
rook-ceph-osd-7-78fcc6f8f6-v5tjs                  2/2     Running     2 (25d ago)     26d
rook-ceph-osd-prepare-node1-j2vbc                 0/1     Completed   0               4h15m
rook-ceph-osd-prepare-node2-xtl7g                 0/1     Completed   0               4h15m
rook-ceph-osd-prepare-node3-dbht6                 0/1     Completed   0               4h15m
rook-ceph-tools-595758779b-twbsw                  1/1     Running     0               160d

进入存储cli
kubectl exec -it rook-ceph-tools-595758779b-twbsw bash -n rook-ceph
ceph status

cluster:
id: 081b8b1d-150c-4472-a0d9-69f9329837ad
health: HEALTH_WARN
mon g is low on available space

services:
mon: 3 daemons, quorum c,f,g (age 12d)
mgr: b(active, since 4h), standbys: a
mds: 1/1 daemons up, 1 hot standby
osd: 8 osds: 8 up (since 3w), 8 in (since 3w)

data:
volumes: 1/1 healthy
pools: 4 pools, 81 pgs
objects: 3.32M objects, 7.6 TiB
usage: 15 TiB used, 62 TiB / 77 TiB avail
pgs: 81 active+clean

io:
client: 29 KiB/s rd, 119 KiB/s wr, 3 op/s rd, 7 op/s wr

ceph osd status

ID  HOST    USED  AVAIL  WR OPS  WR DATA  RD OPS  RD DATA  STATE      
 0  node2  2419G  6704G      0        0       0        0   exists,up  
 1  node3  1454G  9494G      0        0       0        0   exists,up  
 2  node2  2173G  6949G      0        0       1      215   exists,up  
 3  node3  1693G  9255G      0     3163       0      277   exists,up  
 4  node2  2662G  6461G      2     26.1k      1     25.8k  exists,up  
 5  node3  2173G  8775G      1     1933       0      101   exists,up  
 6  node3  1694G  9254G      0        0       0      148   exists,up  
 7  node1  1256G  6194G      0      125       0      169   exists,up 

cat /etc/ceph/ceph.conf

[global]
mon_host = 10.233.45.189:6789,10.233.1.204:6789,10.233.39.171:6789

cat /etc/ceph/keyring

[client.admin]
key = AQDY9IZlOyUTGhAAxwoMGnK/6V54xpjZJobnUQ==

kubectl get svc
kubectl get pod -n genostack-base
kubectl get pod -n genostack-core
kubectl get pod -n genostack-tool

mount cephfs_data
/opt/genostack_service/prod/mountfs.sh

kubectl delete pod postgres-6df8d96d-x92pt -n genostack-base
kubectl delete pod --all -n genostack-base
kubectl delete -f yaml

docker run
-itd
-p 9880:9880
-p 9822:22
-v /data_raid1/gitlab/etc:/etc/gitlab
-v /data_raid1/gitlab/log:/var/log/gitlab
-v /data_raid1/gitlab/opt:/var/opt/gitlab
--privileged=true
twang2218/gitlab-ce-zh:latest

root/Qiankun2021

1. postgre数据库slave01节点：
   启动命令：/home/root/readme.txt

2. 项目启停：
   主节点：
   /opt/service_local/cromwell
   /opt/service_local/qiime_php
   /opt/service_local/qiime_php_new
   /opt/service_local/tesk

   启停任务：
   cd /opt/service_local/cromwell
   kubectl create -f .
   kubectl delete -f .

绿色部分为新版本，单独部署部分；

1. 质量控制 （srynobio/fastp:latest）
    fastp -i SRR11954304.fastq.gz -o SRR11954304_fastp.fastq
2. 去除宿主 （trinityrnaseq/trinityrnaseq:latest）

    bowtie2 -p 15 -x /ceph_disk2/data/tmp/index -U SRR12045797.fastq -S a.sam
    samtools view -bS -F 4 a.sam > a.bam
    samtools sort a.bam -o a.sorted.bam
    samtools index a.sorted.bam
    samtools fastq a.sorted.bam > a.fq 

3. 转录组组装软件（trinityrnaseq/trinityrnaseq:latest）
    Trinity --seqType fq --max_memory 100G --single a.fq


4. 分类 （ncbi/blast:2.12.0）
        (nanozoo/diamond：latest）

    diamond makedb  --in Trinity.fasta -d Trinity
    diamond blastx -d Trinity -q Trinity.fasta -o t1


    makeblastdb -in Trinity.fasta -dbtype prot -title Trinity -out Trinity
    blastx -db test  -query Trinity.fasta -out test.out 
    blastn -db test  -query Trinity.fasta -out test.out 

5. 多序列比对：(staphb/mafft:latest)
    MAFFT 
    mafft --maxiterate 100  sample.fasta > result.fa
    python trans.py result.fa result.phy
6. 系统发育和进化分析(biocontainers/raxml:v8.2.12dfsg-1-deb_cv1)
    RAxML
    raxmlHPC -d -p 12345 -m GTRGAMMAI -s result.phy -n txt

参考文献： https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7929396/#ref33

测试命令

minimap2 -ax sr /ceph_disk1/xinguan/SARS-CoV-2-freebayes/covid19-refseq.fasta SRR11728611.fastp.gz > SRR11728611.sam 
minimap2 -ax sr /ceph_disk1/xinguan/SARS-CoV-2-freebayes/covid19-refseq.fasta SRR11728650.fastp.gz > SRR11728650.sam

samtools view -bS SRR11728611.sam > SRR11728611.bam
samtools view -bS SRR11728650.sam > SRR11728650.bam

samtools sort SRR11728611.sam  > SRR11728611.sorted.bam
samtools sort SRR11728650.sam  > SRR11728650.sorted.bam

samtools index SRR11728611.sorted.bam
samtools index SRR11728650.sorted.bam


freebayes -f /ceph_disk1/xinguan/SARS-CoV-2-freebayes/covid19-refseq.fasta -C 1 SRR11728611.sorted.bam > vcf/SRR11728611.freebayes.vcf
freebayes -f /ceph_disk1/xinguan/SARS-CoV-2-freebayes/covid19-refseq.fasta -C 1 SRR11728650.sorted.bam > vcf/SRR11728650.freebayes.vcf

jacquard merge --include_all ./vcf merged.vcf

问题一： 参考序列格式不规范，在 jacquard merge --include_all ./vcf merged.vcf 会出错
（key error）
错误格式：

NC_045512.2 Severe acute respiratory syndrome coronavirus 2 isolate Wuhan-Hu-1, complete genome
ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAA

修改为：

NC_045512.2 Severe acute respiratory syndrome coronavirus 2 isolate Wuhan-Hu-1
ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAA

官网 ：https://github.com/ComparativeGenomicsToolkit/cactus

一、安装

二、运行

1. step-by-step

1. 打印分步运行的所有命令；

cactus-prepare examples/evolverMammals.txt --outDir steps-output --outSeqFile steps-output/evovlerMammals.txt --outHal steps-output/evolverMammals.hal --jobStore jobstore

2. 简化运行方式 --preprocessOnly

cactus-prepare examples/evolverMammals.txt --outDir steps-output --outSeqFile steps-output/evovlerMammals.txt --outHal steps-output/evolverMammals.hal --jobStore jobstore --preprocessOnly

3. GPU加速 --gpu

The GPU-enabled Docker releases have this turned on by default. It is also possible to manually install it from git and then enable it in cactus via the --configFile cactus option. A template can be found here, and modified to activate the GPU by setting gpuLastz="true" and realign="0" in the <caf> section

cactus-prepare examples/evolverMammals.txt --gpu --outDir steps-output --outSeqFile steps-output/evovlerMammals.txt --outHal steps-output/evolverMammals.hal --jobStore jobstore

有gpu
real 38m50.343s
无gpu
real 52m49.121s

4. 并行优化

并行：只有在独立模块，中并列的命令才可以并列运行，

例1：运行示例中Preprocessor：

1. cactus-preprocess jobstore/0 local.txt steps-output/local.txt --inputNames simHuman_chr6 simCow_chr6 simDog_chr6 --realTimeLogging --logInfo --retryCount 0
2. cactus-preprocess jobstore/1 local.txt steps-output/local.txt --inputNames simMouse_chr6 simRat_chr6 --realTimeLogging --logInfo --retryCount 0

例2：运行示例中Round 0：以下1和2中两部分分别包含三条命令，1和2是并列关系，可并列运行，但 1和2中的三条命令是串行关系，不能并列运行；

1.   cactus-blast jobstore/2 steps-output/local.txt steps-output/Anc2.cigar --root Anc2 --realTimeLogging --logInfo --retryCount 0
cactus-align jobstore/3 steps-output/local.txt steps-output/Anc2.cigar steps-output/Anc2.hal --root Anc2 --realTimeLogging --logInfo --retryCount 0 --maxCores 2
hal2fasta steps-output/Anc2.hal Anc2 --hdf5InMemory > steps-output/Anc2.fa


2.  cactus-blast jobstore/4 steps-output/local.txt steps-output/mr.cigar --root mr --realTimeLogging --logInfo --retryCount 0 
cactus-align jobstore/5 steps-output/local.txt steps-output/mr.cigar steps-output/mr.hal --root mr --realTimeLogging --logInfo --retryCount 0 --maxCores 2
hal2fasta steps-output/Anc1.hal Anc1 --hdf5InMemory > steps-output/Anc1.fa

5.资源选择依据

6. 运行示例

## Preprocessor
cactus-preprocess jobstore/0 local.txt steps-output/local.txt --inputNames simHuman_chr6 simCow_chr6 simDog_chr6 --realTimeLogging --logInfo --retryCount 0


real	0m42.069s
user	1m33.210s
sys	1m49.028s



cactus-preprocess jobstore/1 local.txt steps-output/local.txt --inputNames simMouse_chr6 simRat_chr6 --realTimeLogging --logInfo --retryCount 0 
real	0m39.688s
user	1m4.916s
sys	1m31.333s


## Alignment

### Round 0
cactus-blast jobstore/2 steps-output/local.txt steps-output/Anc2.cigar --root Anc2 --realTimeLogging --logInfo --retryCount 0
real	1m53.030s
user	2m59.048s
sys	3m34.287s

cactus-align jobstore/3 steps-output/local.txt steps-output/Anc2.cigar steps-output/Anc2.hal --root Anc2 --realTimeLogging --logInfo --retryCount 0 --maxCores 2

real	18m22.450s
user	8m34.911s
sys	9m40.047s


hal2fasta steps-output/Anc2.hal Anc2 --hdf5InMemory > steps-output/Anc2.fa
real	0m0.109s
user	0m0.011s
sys	0m0.011s


cactus-blast jobstore/4 steps-output/local.txt steps-output/mr.cigar --root mr --realTimeLogging --logInfo --retryCount 0 
real	1m51.072s
user	3m8.077s
sys	3m15.112s

cactus-align jobstore/5 steps-output/local.txt steps-output/mr.cigar steps-output/mr.hal --root mr --realTimeLogging --logInfo --retryCount 0 --maxCores 2
real	19m22.049s
user	10m10.130s
sys	11m35.772s

hal2fasta steps-output/mr.hal mr --hdf5InMemory > steps-output/mr.fa

real	0m0.019s
user	0m0.015s
sys	0m0.005s


### Round 1
cactus-blast jobstore/6 steps-output/local.txt steps-output/Anc1.cigar --root Anc1 --realTimeLogging --logInfo --retryCount 0 

real	1m19.923s
user	2m9.523s
sys	2m18.150s

-rw-r--r-- 1 root root  325612 Feb  4 06:41 Anc1.cigar.ig_coverage_0
-rw-r--r-- 1 root root  372762 Feb  4 06:41 Anc1.cigar.ig_coverage_1
-rw-r--r-- 1 root root  575910 Feb  4 06:41 Anc1.cigar.og_fragment_0
-rw-r--r-- 1 root root 1623235 Feb  4 06:41 Anc1.cigar.secondary


cactus-align jobstore/7 steps-output/local.txt steps-output/Anc1.cigar steps-output/Anc1.hal --root Anc1 --realTimeLogging --logInfo --retryCount 0 --maxCores 2

real	17m24.211s
user	8m31.867s
sys	10m25.042s

-rw-r--r-- 1 root root  882309 Feb  4 06:41 Anc1.cigar
-rw-r--r-- 1 root root  325612 Feb  4 06:41 Anc1.cigar.ig_coverage_0
-rw-r--r-- 1 root root  372762 Feb  4 06:41 Anc1.cigar.ig_coverage_1
-rw-r--r-- 1 root root  575910 Feb  4 06:41 Anc1.cigar.og_fragment_0
-rw-r--r-- 1 root root 1623235 Feb  4 06:41 Anc1.cigar.secondary
-rw-r--r-- 1 root root 1435545 Feb  4 07:00 Anc1.hal

hal2fasta steps-output/Anc1.hal Anc1 --hdf5InMemory > steps-output/Anc1.fa

real	0m0.018s
user	0m0.011s
sys	0m0.007s


### Round 2
cactus-blast jobstore/8 steps-output/local.txt steps-output/Anc0.cigar --root Anc0 --realTimeLogging --logInfo --retryCount 0 

real	1m13.752s
user	1m22.428s
sys	1m34.542s

-rw-r--r-- 1 root root  211228 Feb  4 07:06 Anc0.cigar
-rw-r--r-- 1 root root  326775 Feb  4 07:06 Anc0.cigar.secondary


cactus-align jobstore/9 steps-output/local.txt steps-output/Anc0.cigar steps-output/local.hal --root Anc0 --realTimeLogging --logInfo --retryCount 0 --maxCores 2

real	13m21.857s
user	5m41.658s
sys	9m4.849s

-rw-r--r-- 1 root root  211228 Feb  4 07:06 Anc0.cigar
-rw-r--r-- 1 root root  326775 Feb  4 07:06 Anc0.cigar.secondary



hal2fasta steps-output/local.hal Anc0 --hdf5InMemory > steps-output/Anc0.fa

real	0m0.018s
user	0m0.018s
sys	0m0.000s


## HAL merging
halAppendSubtree steps-output/local.hal steps-output/Anc1.hal Anc1 Anc1 --merge --hdf5InMemory

-rw-r--r-- 1 root root 2133914 Feb  4 07:23 local.hal

halAppendSubtree steps-output/local.hal steps-output/Anc2.hal Anc2 Anc2 --merge --hdf5InMemory
halAppendSubtree steps-output/local.hal steps-output/mr.hal mr mr --merge --hdf5InMemory

-rw-r--r-- 1 root root 4945057 Feb  4 07:24 local.hal

• 列表客户编写Script来定义一个流程中的每个步骤分别是什么，他们是怎么一步一步相互依赖执行的。Script的语言规范有好几种，Cromwell支持WDL和CWL两种语法。参考：Language Support - Cromwell

• 列表Cromewell Server接收到这个Script后就会解析它，然后根据里面的定义去拆分每个子任务，然后提交到后端的计算资源池去执行。Cromwell称其为Backend，不同的backend实现不同的计算方式。

• 列表Backend计算完毕后将执行结果通知给Cromwell Server。一个流程就完成了。

注意： cromwell 关于Gpu的支持，只支持Google

基于Kubernetes集群的模式

• Volcano

• TESK（不支持GPU参数），若在wdl中添加gpu相关参数，则不生效；会有警告： [WARN] [02/01/2021 05:41:35.404] [cromwell-system-akka.dispatchers.backend-dispatcher-269] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor/WorkflowActor-7139d946-9ff9-4970-af99-01f8c151c38c/WorkflowInitializationActor-7139d946-9ff9-4970-af99-01f8c151c38c/TESK] TESK [UUID(7139d946)]: Key/s [gpuCount, bootDiskSizeGb, gpuType, nvidiaDriverVersion] is/are not supported by backend. Unsupported attributes will not be part of job executions.

* runtime {
* 
*     gpuType: "nvidia-tesla-k80"
* 
*     gpuCount: 2
* 
*     nvidiaDriverVersion: "418.87.00"
* 
*     zones: ["us-central1-c"]
* 
* }