修改文件： /etc/systemd/system/docker.service.d/docker-options.conf

--data-root=/public/docker3
systemctl deamon-reload
systemctl restart docker

拉取镜像

docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kubernetesui_metrics-scraper:v1.0.6
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/library_nginx:1.19
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/coredns:1.7.0
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/dns_k8s-dns-node-cache:1.16.0
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/ingress-nginx_controller:v0.41.2
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-apiserver:v1.19.7
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-controller-manager:v1.19.7
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-proxy:v1.19.7
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-scheduler:v1.19.7
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/pause:3.2
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/pause:3.3
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kubernetesui_dashboard-amd64:v2.1.0
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/cpa_cluster-proportional-autoscaler-amd64:1.8.3
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/calico_cni:v3.16.5
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/calico_kube-controllers:v3.16.5
docker pull dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/calico_node:v3.16.5

docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kubernetesui_metrics-scraper:v1.0.6 docker.io/kubernetesui/metrics-scraper:v1.0.6
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/library_nginx:1.19 docker.io/library/nginx:1.19
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/coredns:1.7.0 k8s.gcr.io/coredns:1.7.0
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/dns_k8s-dns-node-cache:1.16.0 k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/ingress-nginx_controller:v0.41.2 k8s.gcr.io/ingress-nginx/controller:v0.41.2
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-apiserver:v1.19.7 k8s.gcr.io/kube-apiserver:v1.19.7
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-controller-manager:v1.19.7 k8s.gcr.io/kube-controller-manager:v1.19.7
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-proxy:v1.19.7 k8s.gcr.io/kube-proxy:v1.19.7
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kube-scheduler:v1.19.7 k8s.gcr.io/kube-scheduler:v1.19.7
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/pause:3.2 k8s.gcr.io/pause:3.2
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/pause:3.3 k8s.gcr.io/pause:3.3
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/kubernetesui_dashboard-amd64:v2.1.0 docker.io/kubernetesui/dashboard-amd64:v2.1.0
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/cpa_cluster-proportional-autoscaler-amd64:1.8.3 k8s.gcr.io/cpa/cluster-proportional-autoscaler-amd64:1.8.3
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/calico_cni:v3.16.5 quay.io/calico/cni:v3.16.5
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/calico_kube-controllers:v3.16.5 quay.io/calico/kube-controllers:v3.16.5
docker tag dockerhub.genostack.com:8090/k8s/kubernetes-kubespray/calico_node:v3.16.5 quay.io/calico/node:v3.16.5

docker pull dockerhub.genostack.com:8090/rook/cephcsi/cephcsi:v3.7.2
docker pull dockerhub.genostack.com:8090/rook/csi-node-driver-registrar:v2.5.1
docker pull dockerhub.genostack.com:8090/rook/csi-resizer:v1.6.0
docker pull dockerhub.genostack.com:8090/rook/csi-provisioner:v3.3.0
docker pull dockerhub.genostack.com:8090/rook/csi-snapshotter:v6.1.0
docker pull dockerhub.genostack.com:8090/rook/csi-attacher:v4.0.0
docker pull dockerhub.genostack.com:8090/rook/ceph/ceph:v17.2.5

docker tag dockerhub.genostack.com:8090/rook/cephcsi/cephcsi:v3.7.2 quay.io/cephcsi/cephcsi:v3.7.2
docker tag dockerhub.genostack.com:8090/rook/csi-node-driver-registrar:v2.5.1 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
docker tag dockerhub.genostack.com:8090/rook/csi-resizer:v1.6.0 registry.k8s.io/sig-storage/csi-resizer:v1.6.0
docker tag dockerhub.genostack.com:8090/rook/csi-provisioner:v3.3.0 registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
docker tag dockerhub.genostack.com:8090/rook/csi-snapshotter:v6.1.0 registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0
docker tag dockerhub.genostack.com:8090/rook/csi-attacher:v4.0.0 registry.k8s.io/sig-storage/csi-attacher:v4.0.0
docker tag dockerhub.genostack.com:8090/rook/ceph/ceph:v17.2.5 quay.io/ceph/ceph:v17.2.5
docker pull dockerhub.genostack.com:8090/library/ceph:v1.10.8
docker tag dockerhub.genostack.com:8090/library/ceph:v1.10.8 rook/ceph:v1.10.8

重启kubelet

systemctl restart kubelet

等待一会后：执行

docker ps

会有很多k8s的pod启动

问题二： kubelet起不来，

启动时有时会检测: /etc/kubernetes/bootstrap-kubelet.conf,但这个文件不知道为啥会被自动删除，

此时需要复制一份数据：

cp /etc/kubernetes/admin.conf /etc/kubernetes/bootstrap-kubelet.conf
systemctl restart kubelet

所有节点都启动好了后，查看集群状态
kubectl get node
kubectl get pod -n kube-system
查看ceph
kubectl get pod -n rook-ceph
kubectl exec -it rook-ceph-tools-6867866dddsss-h5tc4 bash -n rook-ceph
执行ceph status
volumes: 1/1 healthy，则存储正常

挂载cephfs_data 路径, node1
/opt/genostack_service/prod/mount.sh

如果挂载不上，进入pod
kubectl exec -it rook-ceph-tools-6867866dddsss-h5tc4 bash -n rook-ceph
使用最新的key和host
cat /etc/ceph/keyring
cat /etc/ceph/ceph.conf

重启平台服务，重启前需要检查挂载naSS
kubectl delete pod -n genostack-base
kubectl delete pod -n genostack-core

问题三：kubesphere问题：
重建docker路径后会缺少镜像: 在缺少镜像的节点执行
/cephfs_data/images/kubesphere/load.sh

kubesphere 默认资源不够也会导致启动失败， 修改limit: 10Gi
kubectl get deployment -n kubesphere-system
kubectl edit ks-apiserver -n kubesphere-system
kubectl edit ks-console -n kubesphere-system
kubectl edit ks-controller-manager -n kubesphere-system

问题四挂载nfs路径：
mount -t nfs 192.168.30.158:/nfs-share/ProductionData/Production\ Order /public/public158/
mount -t nfs 192.168.30.212:/volume1/k8s_bak/ /public/public212/