晶能项目安装记录

[zhanglu]

晶能项目：

1. 安装操作系统ubuntu22.04
2. 安装k8s(kubespray-2.24.1+)
3. 向日葵或者远程环境
4. kubesphere管理软件
5. genostack(php+cromwell)
6. jupyter

######### 安装节点执行
apt install python3.10-venv
mkdir /opt/app
cd /opt/app
wget https://file.genostack.com/public_resource/kubespray_2_24_1/kubespray-2.24.1.tar.gz
tar -zxvf kubespray-2.24.1.tar.gz
cd kubespray-2.24.1

VENVDIR=kubespray-venv
KUBESPRAYDIR=/opt/app/kubespray-2.24.1/
python3 -m venv $VENVDIR
source $VENVDIR/bin/activate
cd $KUBESPRAYDIR
pip install -U -r requirements.txt

cd /tmp
wget https://file.genostack.com/public_resource/kubespray_2_24_1/releases.tar.gz
tar -zxvf releases.tar.gz

cd /opt/app/kubespray-2.24.1/
cp -rpf inventory/sample inventory/mycluster
USE_exportREAL_HOSTNAME=true
export CONFIG_FILE=inventory/mycluster/hosts.yaml
declare -a IPS=(10.155.19.223 10.155.19.64 10.155.19.147)
python3 contrib/inventory_builder/inventory.py ${IPS[@]}

######### k8s-node执行
setenforce 0
sed -i '/SELINUX/s/enforcing/disabled/' /etc/selinux/config
systemctl stop firewalld && systemctl disable firewalld
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat && iptables -P FORWARD ACCEPT
swapoff -a && free –h
service dnsmasq stop && systemctl disable dnsmasq

cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
vm.overcommit_memory = 1
EOF

sysctl -p /etc/sysctl.d/kubernetes.conf

修改node性质
vi inventory/mycluster/hosts.yaml
ansible-playbook -i inventory/mycluster/hosts.yaml -b cluster.yml -vvvv

安装十分钟左右会报错，无法下载镜像
修改容器地址

##########################################

version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
oom_score = 0

[grpc]
max_recv_message_size = 16777216
max_send_message_size = 16777216

[debug]
level = "info"

[metrics]
address = ""
grpc_histogram = false

[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.9"
max_container_log_line_size = -1
enable_unprivileged_ports = false
enable_unprivileged_icmp = false
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
runtime_engine = ""
runtime_root = ""
base_runtime_spec = "/etc/containerd/cri-base.json"

      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
        systemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
      endpoint = ["https://registry-1.docker.io"]

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."dockerhub.genostack.com:8090"]
      endpoint = ["http://dockerhub.genostack.com:8090"]
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."dockerhub.genostack.com:8090".tls]
      insecure_skip_verify = true
    [plugins."io.containerd.grpc.v1.cri".registry.configs."dockerhub.genostack.com:8090".auth]
      username = "admin"
      password = "Qiankun2021"

#################################################
修改后重启containerd
systemctl restart containerd

nerdctl pull dyrnq/pause:3.9
nerdctl pull dyrnq/coredns:v1.10.1
nerdctl pull dyrnq/k8s-dns-node-cache:1.22.28
nerdctl pull dyrnq/metrics-server:v0.6.4
nerdctl pull dyrnq/local-volume-provisioner:v2.5.0
nerdctl pull dyrnq/controller:v1.8.1
nerdctl pull dyrnq/csi-attacher:v3.3.0
nerdctl pull dyrnq/csi-provisioner:v3.0.0
nerdctl pull dyrnq/csi-snapshotter:v5.0.0
nerdctl pull dyrnq/csi-resizer:v1.3.0
nerdctl pull dyrnq/csi-node-driver-registrar:v2.4.0
nerdctl pull dyrnq/livenessprobe:v2.5.0
nerdctl pull dockergroo/cluster-proportional-autoscaler:v1.8.8

nerdctl tag dyrnq/pause:3.9 registry.k8s.io/pause:3.9
nerdctl tag dyrnq/pause:3.9 registry.k8s.io/kube-proxy/pause:3.9
nerdctl tag dyrnq/coredns:v1.10.1 registry.k8s.io/coredns/coredns:v1.10.1
nerdctl tag dyrnq/k8s-dns-node-cache:1.22.28 registry.k8s.io/dns/k8s-dns-node-cache:1.22.28
nerdctl tag dyrnq/metrics-server:v0.6.4 registry.k8s.io/metrics-server/metrics-server:v0.6.4
nerdctl tag dyrnq/local-volume-provisioner:v2.5.0 registry.k8s.io/sig-storage/local-volume-provisioner:v2.5.0
nerdctl tag dyrnq/controller:v1.8.1 registry.k8s.io/ingress-nginx/controller:v1.8.1
nerdctl tag dyrnq/csi-attacher:v3.3.0 registry.k8s.io/sig-storage/csi-attacher:v3.3.0
nerdctl tag dyrnq/csi-provisioner:v3.0.0 registry.k8s.io/sig-storage/csi-provisioner:v3.0.0
nerdctl tag dyrnq/csi-snapshotter:v5.0.0 registry.k8s.io/sig-storage/csi-snapshotter:v5.0.0
nerdctl tag dyrnq/csi-resizer:v1.3.0 registry.k8s.io/sig-storage/csi-resizer:v1.3.0
nerdctl tag dyrnq/csi-node-driver-registrar:v2.4.0 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0
nerdctl tag dyrnq/livenessprobe:v2.5.0 registry.k8s.io/sig-storage/livenessprobe:v2.5.0
nerdctl tag dockergroo/cluster-proportional-autoscaler:v1.8.8 registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.8.8

nerdctl pull dyrnq/kube-apiserver:v1.28.6
nerdctl tag dyrnq/kube-apiserver:v1.28.6 registry.k8s.io/kube-apiserver:v1.28.6
nerdctl pull dyrnq/kube-controller-manager:v1.28.6
nerdctl tag dyrnq/kube-controller-manager:v1.28.6 registry.k8s.io/kube-controller-manager:v1.28.6
nerdctl pull dyrnq/kube-scheduler:v1.28.6
nerdctl tag dyrnq/kube-scheduler:v1.28.6 registry.k8s.io/kube-scheduler:v1.28.6
nerdctl pull dyrnq/kube-proxy:v1.28.6
nerdctl tag dyrnq/kube-proxy:v1.28.6 registry.k8s.io/kube-proxy:v1.28.6

安装slurm
cd /opt/app
wget https://file.genostack.com/public_resource/slurm_install.zip
unzip slurm_install.zip

cd slurm_install

[zhanglu]

https://file.genostack.com/public_resource/genostack_v3_service.tar.gz
https://file.genostack.com/public_resource/genostack_v3_slurm.tar.gz

[zhanglu]

https://file.genostack.com/public_resource/local-path-storage.yaml