NAME    STATUS   ROLES           AGE    VERSION
node1   Ready    control-plane   169d   v1.27.7
node2   Ready    control-plane   169d   v1.27.7
node3   Ready    <none>          169d   v1.27.7

kubectl get namespace

kubectl get pod -n kube-system

calico-node-hgmst                          1/1     Running   7 (65d ago)     168d
calico-node-kmmfq                          1/1     Running   0               25d
calico-node-l52r8                          1/1     Running   1 (167d ago)    168d
coredns-5c469774b8-29czz                   1/1     Running   0               26d
coredns-5c469774b8-4qh9l                   1/1     Running   0               165d
dns-autoscaler-f455cf558-lf5jb             1/1     Running   0               163d
kube-apiserver-node1                       1/1     Running   87 (25d ago)    26d
kube-apiserver-node2                       1/1     Running   226 (65d ago)   169d
kube-controller-manager-node1              1/1     Running   83 (25d ago)    25d
kube-controller-manager-node2              1/1     Running   39 (12d ago)    169d
kube-proxy-25qlx                           1/1     Running   1 (167d ago)    168d
kube-proxy-7kjtt                           1/1     Running   9 (25d ago)     168d
kube-proxy-h5mkk                           1/1     Running   7 (65d ago)     168d
kube-scheduler-node1                       1/1     Running   72 (25d ago)    25d
kube-scheduler-node2                       1/1     Running   37 (12d ago)    169d
metrics-server-66549d4ff5-hs6zn            1/1     Running   0               24d
nginx-proxy-node3                          1/1     Running   183 (65d ago)   169d
nodelocaldns-f4mtg                         1/1     Running   504 (64d ago)   168d
nodelocaldns-n47wf                         1/1     Running   98 (25d ago)    168d
nodelocaldns-xhqj9                         1/1     Running   908 (64d ago)   168d
snapshot-controller-0                      1/1     Running   0               25d

kubectl get pod -n rook-ceph

NAME                                              READY   STATUS      RESTARTS        AGE
csi-cephfsplugin-6t2lg                            2/2     Running     0               164d
csi-cephfsplugin-bcj2v                            2/2     Running     24 (25d ago)    164d
csi-cephfsplugin-provisioner-5b59dbc849-8n879     5/5     Running     47 (21h ago)    163d
csi-cephfsplugin-provisioner-5b59dbc849-vcttq     5/5     Running     4 (7d21h ago)   26d
csi-cephfsplugin-zjbvt                            2/2     Running     12 (65d ago)    164d
csi-rbdplugin-h76b8                               2/2     Running     12 (65d ago)    164d
csi-rbdplugin-n85dj                               2/2     Running     23 (25d ago)    164d
csi-rbdplugin-provisioner-59777f956d-8zt6f        5/5     Running     0               26d
csi-rbdplugin-provisioner-59777f956d-tdz77        5/5     Running     21 (65d ago)    163d
csi-rbdplugin-wr94r                               2/2     Running     0               164d
rook-ceph-crashcollector-node1-6797f99845-6mnxf   1/1     Running     2 (25d ago)     26d
rook-ceph-crashcollector-node2-6599d4856-wzmqp    1/1     Running     0               155d
rook-ceph-crashcollector-node3-7769d57785-wk7t7   1/1     Running     0               56d
rook-ceph-exporter-node1-b5589c46f-nxct7          1/1     Running     3 (25d ago)     26d
rook-ceph-exporter-node2-756fc8668-zv7wg          1/1     Running     0               155d
rook-ceph-exporter-node3-5c5ddc477c-q82rq         1/1     Running     0               56d
rook-ceph-mds-myfs-ec-a-847b49d7fd-njctk          2/2     Running     5 (26d ago)     56d
rook-ceph-mds-myfs-ec-b-f676f7fbd-4r5vg           2/2     Running     82 (26d ago)    155d
rook-ceph-mgr-a-bf8f784f9-7m4cw                   3/3     Running     0               26d
rook-ceph-mgr-b-59f8d89bfb-fkshl                  3/3     Running     0               160d
rook-ceph-mon-c-84f75f85d8-h2mv4                  2/2     Running     0               164d
rook-ceph-mon-f-687bf74f58-chqqj                  2/2     Running     0               64d
rook-ceph-mon-g-55fd64cdc7-kch85                  2/2     Running     4 (25d ago)     25d
rook-ceph-operator-544c5c6477-qkm2q               1/1     Running     0               160d
rook-ceph-osd-0-774465f4c7-sljpn                  2/2     Running     0               164d
rook-ceph-osd-1-795795559b-4h92l                  2/2     Running     0               65d
rook-ceph-osd-2-7f69b64959-pqt65                  2/2     Running     0               164d
rook-ceph-osd-3-5d458b5794-jfczk                  2/2     Running     0               65d
rook-ceph-osd-4-5bbc59d7cd-n5wtn                  2/2     Running     0               164d
rook-ceph-osd-5-894df5d9f-274jw                   2/2     Running     0               65d
rook-ceph-osd-6-77bcc66bb-4gvnn                   2/2     Running     0               65d
rook-ceph-osd-7-78fcc6f8f6-v5tjs                  2/2     Running     2 (25d ago)     26d
rook-ceph-osd-prepare-node1-j2vbc                 0/1     Completed   0               4h15m
rook-ceph-osd-prepare-node2-xtl7g                 0/1     Completed   0               4h15m
rook-ceph-osd-prepare-node3-dbht6                 0/1     Completed   0               4h15m
rook-ceph-tools-595758779b-twbsw                  1/1     Running     0               160d

进入存储cli
kubectl exec -it rook-ceph-tools-595758779b-twbsw bash -n rook-ceph
ceph status

cluster:
id: 081b8b1d-150c-4472-a0d9-69f9329837ad
health: HEALTH_WARN
mon g is low on available space

services:
mon: 3 daemons, quorum c,f,g (age 12d)
mgr: b(active, since 4h), standbys: a
mds: 1/1 daemons up, 1 hot standby
osd: 8 osds: 8 up (since 3w), 8 in (since 3w)

data:
volumes: 1/1 healthy
pools: 4 pools, 81 pgs
objects: 3.32M objects, 7.6 TiB
usage: 15 TiB used, 62 TiB / 77 TiB avail
pgs: 81 active+clean

io:
client: 29 KiB/s rd, 119 KiB/s wr, 3 op/s rd, 7 op/s wr

ceph osd status

ID  HOST    USED  AVAIL  WR OPS  WR DATA  RD OPS  RD DATA  STATE      
 0  node2  2419G  6704G      0        0       0        0   exists,up  
 1  node3  1454G  9494G      0        0       0        0   exists,up  
 2  node2  2173G  6949G      0        0       1      215   exists,up  
 3  node3  1693G  9255G      0     3163       0      277   exists,up  
 4  node2  2662G  6461G      2     26.1k      1     25.8k  exists,up  
 5  node3  2173G  8775G      1     1933       0      101   exists,up  
 6  node3  1694G  9254G      0        0       0      148   exists,up  
 7  node1  1256G  6194G      0      125       0      169   exists,up 

cat /etc/ceph/ceph.conf

[global]
mon_host = 10.233.45.189:6789,10.233.1.204:6789,10.233.39.171:6789

cat /etc/ceph/keyring

[client.admin]
key = AQDY9IZlOyUTGhAAxwoMGnK/6V54xpjZJobnUQ==

kubectl get svc
kubectl get pod -n genostack-base
kubectl get pod -n genostack-core
kubectl get pod -n genostack-tool

mount cephfs_data
/opt/genostack_service/prod/mountfs.sh

kubectl delete pod postgres-6df8d96d-x92pt -n genostack-base
kubectl delete pod --all -n genostack-base
kubectl delete -f yaml