From 6b5474a48d6e89d28ffd35b99f1e460845ea1820 Mon Sep 17 00:00:00 2001 From: Vassiliy Yegorov Date: Tue, 21 Feb 2023 14:35:39 +0700 Subject: [PATCH] init --- 1.Plan/README.md | 2 +- 2.Nodes/README.md | 15 +++ 3.RKE2/README.md | 104 ++++++++++++++++-- 3.RKE2/nfs/nfs-install.sh | 16 --- 3.RKE2/traefik/.env.example | 1 + 3.RKE2/traefik/.gitignore | 1 + 3.RKE2/traefik/.gitkeep | 0 3.RKE2/traefik/data/custom/example/app.yaml | 17 +++ 3.RKE2/traefik/data/custom/rke-custom.yaml | 29 +++++ 3.RKE2/traefik/data/traefik.yaml | 39 +++++++ 3.RKE2/traefik/docker-compose.yml | 44 ++++++++ 3.RKE2/traefik/init.sh | 7 ++ 4.NFS/README.md | 18 +++ 4.NFS/nfs/nfs-install.sh | 5 + .../nfs/nfs-provisioner/Chart.yaml | 0 .../nfs/nfs-provisioner/README.md | 0 .../nfs/nfs-provisioner/ci/test-values.yaml | 0 .../nfs-provisioner/templates/_helpers.tpl | 0 .../templates/clusterrole.yaml | 0 .../templates/clusterrolebinding.yaml | 0 .../nfs-provisioner/templates/deployment.yaml | 0 .../templates/persistentvolume.yaml | 0 .../templates/persistentvolumeclaim.yaml | 0 .../templates/podsecuritypolicy.yaml | 0 .../nfs/nfs-provisioner/templates/role.yaml | 0 .../templates/rolebinding.yaml | 0 .../templates/serviceaccount.yaml | 0 .../templates/storageclass.yaml | 0 .../nfs/nfs-provisioner/values.yaml | 2 +- {3.RKE2 => 4.NFS}/nfs/nfs-server.sh | 2 +- 30 files changed, 276 insertions(+), 26 deletions(-) delete mode 100755 3.RKE2/nfs/nfs-install.sh create mode 100644 3.RKE2/traefik/.env.example create mode 100644 3.RKE2/traefik/.gitignore create mode 100644 3.RKE2/traefik/.gitkeep create mode 100644 3.RKE2/traefik/data/custom/example/app.yaml create mode 100644 3.RKE2/traefik/data/custom/rke-custom.yaml create mode 100644 3.RKE2/traefik/data/traefik.yaml create mode 100644 3.RKE2/traefik/docker-compose.yml create mode 100644 3.RKE2/traefik/init.sh create mode 100644 4.NFS/README.md create mode 100755 4.NFS/nfs/nfs-install.sh rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/Chart.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/README.md (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/ci/test-values.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/_helpers.tpl (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/clusterrole.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/clusterrolebinding.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/deployment.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/persistentvolume.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/persistentvolumeclaim.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/podsecuritypolicy.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/role.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/rolebinding.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/serviceaccount.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/templates/storageclass.yaml (100%) rename {3.RKE2 => 4.NFS}/nfs/nfs-provisioner/values.yaml (98%) rename {3.RKE2 => 4.NFS}/nfs/nfs-server.sh (64%) diff --git a/1.Plan/README.md b/1.Plan/README.md index 5a7c4d2..549f7b9 100644 --- a/1.Plan/README.md +++ b/1.Plan/README.md @@ -3,5 +3,5 @@ что нам понадобиться: 1. развернутый [proxmox](https://www.proxmox.com/en/downloads) (подойдет любой гипервизор на вашей машине, vmware workstation например) -2. ISO-образ ubuntu [20.04](https://releases.ubuntu.com) (можно 22.04) +2. Cloud ISO-образ ubuntu [22.04](https://cloud-images.ubuntu.com) (можно 20.04) 3. [Lens](https://k8slens.dev) diff --git a/2.Nodes/README.md b/2.Nodes/README.md index 06b0e6f..b612563 100644 --- a/2.Nodes/README.md +++ b/2.Nodes/README.md @@ -6,3 +6,18 @@ 2. Настраиваем сеть и необходимые пакеты 3. Шаблонизируем 4. Разворачиваем из шаблона 3 ВМ, прописываем имена хостов и статичную сеть + +## Готовим шаблон (действия на гипере) + +1. wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img +2. qm create 1000 --memory 4096 --net0 virtio,bridge=vmbr20 --agent 1 +3. qm importdisk 1000 jammy-server-cloudimg-amd64.img data +4. qm set 1000 --scsihw virtio-scsi-pci --scsi0 data:vm-1000-disk-0 +5. qm set 1000 --ide2 data:cloudinit +6. qm set 1000 --boot c --bootdisk scsi0 + +7. настраиваем базовый CloudInit (логин, пароль, ключ, dhcp) +8. запускаем машину +9. ставим пакеты, украшаем консоль +10. не забываем `apt install qemu-guest-agent` +11. шаблонизируем diff --git a/3.RKE2/README.md b/3.RKE2/README.md index 0760727..994ce70 100644 --- a/3.RKE2/README.md +++ b/3.RKE2/README.md @@ -4,16 +4,106 @@ * [helm](https://helm.sh) -План действий: +## План действий 1. В чем разница между RKE <> RKE2 2. Снапшотим наши ноды, чтоб можно было откатиться к началу 3. Запускаем RKE2 через комм строку 4. Тестируем кластер в линзе 5. Восстанавливаемся из снепшотов -6. Запускаем +1 ВМ для ранчера, ставим на ней docker -7. Поднимаем rancher -8. Разворачиваем кластер rke через rancher, проверяем его в линзе -9. Запускаем контейнер для NFS -10. Запускаем в кластере SNI для NFS-сервера -11. Подключаем метрики от линзы + +## Выполнение + +на первой ноде: + +```bash +curl -sfL https://get.rke2.io | sh - +systemctl enable --now rke2-server.service +``` + +Подключаем "кластер" к линзе, взяв конфиг из `/etc/rancher/rke2/rke2.yaml` и поправив там +верный ip-адрес. + +на второи и третьей: + +```bash +mkdir -p /etc/rancher/rke2/ && nano /etc/rancher/rke2/config.yaml +``` + +и заполняем его конфигом: +(токен берем из первой ноды в файле `/var/lib/rancher/rke2/server/node-token`) + +```yaml +server: https://192.168.20.2:9345 +token: K10a251c5c4fc92ca9d08d3aa534e232bd9cf1f6c192d42551c98020ffec17464cc::server:1b306239dd7e878eda35084137aa9626 +tls-san: + - rke.bildme.ru +node-taint: + - "CriticalAddonsOnly=true:NoExecute" +``` + +## Траефик + +запускаем траефик, балансирую только на первую cp-ноду + +траефик у меня будет висеть на адресе 192.168.20.2, от сюда этот адрес указан и в конфиге в разделе server + +## Подключаем 2\3 ноды + +активируем сервис, ждем подключения нод в кластер + +`systemctl enable --now rke2-server.service` + +## Правим подключение + +1. идем на первую ноду, так же создаем конфиг, сохраняем и перезапускаем машину +2. перенастраиваем подключение в линзе по доменному имени +3. открываем балансировку в траефике на все три ноды + +## Добавляем воркер-ноду + +1. качаем бинари - `curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh -` +2. создаем более простой конфиг: + +```bash +mkdir -p /etc/rancher/rke2/ && nano /etc/rancher/rke2/config.yaml +``` + +заполняем: + +```yaml +server: https://192.168.20.2:9345 +token: K10a251c5c4fc92ca9d08d3aa534e232bd9cf1f6c192d42551c98020ffec17464cc::server:1b306239dd7e878eda35084137aa9626 +``` + +запускаем службу: + +`systemctl enable --now rke2-agent.service` + +ждем адопта, затем добавляем в лейбы ноды значение (исключительно для красоты): + +`node-role.kubernetes.io/worker: 'true'` + +## Со стороны кластера + +1. поправим taints на первой ноде, так, что бы на нее не пошла боевая нагрузка +2. поправим nginx ingress controller, что бы он работал только на CP-нодах + +меняем селектор нод: + +```yaml +nodeSelector: + node-role.kubernetes.io/control-plane: 'true' +``` + +добавляем разрешение на запуск на CP-нодах: + +```yaml + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute +``` + +проверяем, что у нас запущены 3 реплики rke2-ingress-nginx-controller diff --git a/3.RKE2/nfs/nfs-install.sh b/3.RKE2/nfs/nfs-install.sh deleted file mode 100755 index 6e4eea3..0000000 --- a/3.RKE2/nfs/nfs-install.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -ssh node1 apt install -y nfs-common -ssh node1 systemctl enable rpcbind -ssh node1 systemctl start rpcbind - -ssh node2 apt install -y nfs-common -ssh node2 systemctl enable rpcbind -ssh node2 systemctl start rpcbind - -ssh node3 apt install -y nfs-common -ssh node3 systemctl enable rpcbind -ssh node3 systemctl start rpcbind - - -#helm upgrade --install --create-namespace nfs-provisioner -n nfs-provisioner nfs-provisioner diff --git a/3.RKE2/traefik/.env.example b/3.RKE2/traefik/.env.example new file mode 100644 index 0000000..be6ef4b --- /dev/null +++ b/3.RKE2/traefik/.env.example @@ -0,0 +1 @@ +HOSTNAME=traefik.domain.ru diff --git a/3.RKE2/traefik/.gitignore b/3.RKE2/traefik/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/3.RKE2/traefik/.gitignore @@ -0,0 +1 @@ +.env diff --git a/3.RKE2/traefik/.gitkeep b/3.RKE2/traefik/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/3.RKE2/traefik/data/custom/example/app.yaml b/3.RKE2/traefik/data/custom/example/app.yaml new file mode 100644 index 0000000..6ac12ec --- /dev/null +++ b/3.RKE2/traefik/data/custom/example/app.yaml @@ -0,0 +1,17 @@ +http: + routers: + app-domain-ru-route: + entryPoints: + - https + service: app-domain-ru-service + rule: Host(`$APP_HOSTNAME`) + tls: + certResolver: letsEncrypt + services: + app-domain-ru-service: + loadBalancer: + passHostHeader: true + servers: + - url: http://192.168.20.101 + - url: http://192.168.20.102 + - url: http://192.168.20.103 diff --git a/3.RKE2/traefik/data/custom/rke-custom.yaml b/3.RKE2/traefik/data/custom/rke-custom.yaml new file mode 100644 index 0000000..da514c2 --- /dev/null +++ b/3.RKE2/traefik/data/custom/rke-custom.yaml @@ -0,0 +1,29 @@ +tcp: + routers: + rke2-api: + entryPoints: + - k8s-api + rule: "HostSNI(`*`)" + service: rke2-api-service + tls: + passthrough: true + rke2-connect: + entryPoints: + - rke2-connect + rule: "HostSNI(`*`)" + service: rke2-connect-service + tls: + passthrough: true + services: + rke2-api-service: + loadBalancer: + servers: + - address: 192.168.20.101:6443 + # - address: 192.168.20.102:6443 + # - address: 192.168.20.103:6443 + rke2-connect-service: + loadBalancer: + servers: + - address: 192.168.20.101:9345 + # - address: 192.168.20.102:9345 + # - address: 192.168.20.103:9345 diff --git a/3.RKE2/traefik/data/traefik.yaml b/3.RKE2/traefik/data/traefik.yaml new file mode 100644 index 0000000..a638774 --- /dev/null +++ b/3.RKE2/traefik/data/traefik.yaml @@ -0,0 +1,39 @@ +global: + checkNewVersion: true + +log: + level: error + filePath: /data/stdout.log + format: common + +serversTransport: + insecureSkipVerify: true + +api: + dashboard: true + +entryPoints: + http: + address: ":80" + https: + address: ":443" + k8s-api: + address: ":6443" + rke2-connect: + address: ":9345" + +providers: + docker: + endpoint: "unix:///var/run/docker.sock" + exposedByDefault: false + file: + directory: /custom + watch: true + +certificatesResolvers: + letsEncrypt: + acme: + email: mail@gmail.com + storage: acme.json + httpChallenge: + entryPoint: http diff --git a/3.RKE2/traefik/docker-compose.yml b/3.RKE2/traefik/docker-compose.yml new file mode 100644 index 0000000..04a339c --- /dev/null +++ b/3.RKE2/traefik/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3.9' +services: + traefik: + image: traefik + container_name: traefik + restart: unless-stopped + security_opt: + - no-new-privileges:true + ports: + - 80:80 + - 443:443 + - 6443:6443 + - 9345:9345 + extra_hosts: + kubernetes.default: 127.0.0.1 + cap_add: + - NET_BIND_SERVICE + volumes: + - /etc/localtime:/etc/localtime:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./data/traefik.yml:/traefik.yml:ro + - ./data/custom/:/custom/:ro + - ./data/acme.json:/acme.json + - ./logs/stdout.log:/data/stdout.log:rw + - ./logs/access.log:/data/access.log:rw + labels: + - "traefik.enable=true" + - "traefik.http.routers.traefik.entrypoints=https" + - "traefik.http.routers.traefik.rule=Host(`$HOSTNAME`)" + - "traefik.http.routers.traefik.tls=true" + - "traefik.http.routers.traefik.tls.certresolver=letsEncrypt" + - "traefik.http.routers.traefik.service=api@internal" + - "traefik.http.services.traefik-traefik.loadbalancer.server.port=888" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + # global redirect to https + - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.entrypoints=http" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https" + networks: + - webproxy + +networks: + webproxy: + name: webproxy diff --git a/3.RKE2/traefik/init.sh b/3.RKE2/traefik/init.sh new file mode 100644 index 0000000..3cc0ec8 --- /dev/null +++ b/3.RKE2/traefik/init.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +touch data/acme.json +chmod 600 data/acme.json + +touch logs/stdout.log +touch logs/access.log diff --git a/4.NFS/README.md b/4.NFS/README.md new file mode 100644 index 0000000..5b5f0f4 --- /dev/null +++ b/4.NFS/README.md @@ -0,0 +1,18 @@ +# подключаем NFS и StorageClass + +1. разворачиваем контейнер +2. поднимаем на нем nfs-server +3. на нодах просаживаем пакет для работы с маунтингом дисков +4. в кластере запускаем helm +5. деплоим метрики в линзе +6. правим толерантность в node-exporter, добавляя: + +```yaml + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute +``` + +done. diff --git a/4.NFS/nfs/nfs-install.sh b/4.NFS/nfs/nfs-install.sh new file mode 100755 index 0000000..9b31aa4 --- /dev/null +++ b/4.NFS/nfs/nfs-install.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +ssh root@192.168.20.110 "apt install -y nfs-common && systemctl enable --now rpcbind" + +#helm upgrade --install --create-namespace nfs-provisioner -n nfs-provisioner nfs-provisioner diff --git a/3.RKE2/nfs/nfs-provisioner/Chart.yaml b/4.NFS/nfs/nfs-provisioner/Chart.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/Chart.yaml rename to 4.NFS/nfs/nfs-provisioner/Chart.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/README.md b/4.NFS/nfs/nfs-provisioner/README.md similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/README.md rename to 4.NFS/nfs/nfs-provisioner/README.md diff --git a/3.RKE2/nfs/nfs-provisioner/ci/test-values.yaml b/4.NFS/nfs/nfs-provisioner/ci/test-values.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/ci/test-values.yaml rename to 4.NFS/nfs/nfs-provisioner/ci/test-values.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/_helpers.tpl b/4.NFS/nfs/nfs-provisioner/templates/_helpers.tpl similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/_helpers.tpl rename to 4.NFS/nfs/nfs-provisioner/templates/_helpers.tpl diff --git a/3.RKE2/nfs/nfs-provisioner/templates/clusterrole.yaml b/4.NFS/nfs/nfs-provisioner/templates/clusterrole.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/clusterrole.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/clusterrole.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/clusterrolebinding.yaml b/4.NFS/nfs/nfs-provisioner/templates/clusterrolebinding.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/clusterrolebinding.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/clusterrolebinding.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/deployment.yaml b/4.NFS/nfs/nfs-provisioner/templates/deployment.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/deployment.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/deployment.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/persistentvolume.yaml b/4.NFS/nfs/nfs-provisioner/templates/persistentvolume.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/persistentvolume.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/persistentvolume.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/persistentvolumeclaim.yaml b/4.NFS/nfs/nfs-provisioner/templates/persistentvolumeclaim.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/persistentvolumeclaim.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/persistentvolumeclaim.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/podsecuritypolicy.yaml b/4.NFS/nfs/nfs-provisioner/templates/podsecuritypolicy.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/podsecuritypolicy.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/podsecuritypolicy.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/role.yaml b/4.NFS/nfs/nfs-provisioner/templates/role.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/role.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/role.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/rolebinding.yaml b/4.NFS/nfs/nfs-provisioner/templates/rolebinding.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/rolebinding.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/rolebinding.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/serviceaccount.yaml b/4.NFS/nfs/nfs-provisioner/templates/serviceaccount.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/serviceaccount.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/serviceaccount.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/templates/storageclass.yaml b/4.NFS/nfs/nfs-provisioner/templates/storageclass.yaml similarity index 100% rename from 3.RKE2/nfs/nfs-provisioner/templates/storageclass.yaml rename to 4.NFS/nfs/nfs-provisioner/templates/storageclass.yaml diff --git a/3.RKE2/nfs/nfs-provisioner/values.yaml b/4.NFS/nfs/nfs-provisioner/values.yaml similarity index 98% rename from 3.RKE2/nfs/nfs-provisioner/values.yaml rename to 4.NFS/nfs/nfs-provisioner/values.yaml index 98c2c3d..ef2aa09 100644 --- a/3.RKE2/nfs/nfs-provisioner/values.yaml +++ b/4.NFS/nfs/nfs-provisioner/values.yaml @@ -7,7 +7,7 @@ image: pullPolicy: IfNotPresent nfs: - server: 192.168.9.99 + server: 192.168.20.99 path: /mnt/data mountOptions: diff --git a/3.RKE2/nfs/nfs-server.sh b/4.NFS/nfs/nfs-server.sh similarity index 64% rename from 3.RKE2/nfs/nfs-server.sh rename to 4.NFS/nfs/nfs-server.sh index 92b2e29..1fb1eba 100644 --- a/3.RKE2/nfs/nfs-server.sh +++ b/4.NFS/nfs/nfs-server.sh @@ -7,6 +7,6 @@ FOLDER=/mnt/data apt update && apt install -y nfs-kernel-server chown nobody:nogroup $FOLDER -echo "$FOLDER 192.168.9.0/24(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports +echo "$FOLDER 192.168.20.0/24(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports systemctl restart nfs-kernel-server