From 54486437429a15b2147038bed34915d768347fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Jochum?= Date: Fri, 28 Aug 2020 02:52:01 +0200 Subject: [PATCH] Update rancher-k3s Doku --- content/post/rancher-k3s-with-galera.md | 202 ++++++++++++++++++++---- 1 file changed, 172 insertions(+), 30 deletions(-) diff --git a/content/post/rancher-k3s-with-galera.md b/content/post/rancher-k3s-with-galera.md index ea2921e..2544682 100644 --- a/content/post/rancher-k3s-with-galera.md +++ b/content/post/rancher-k3s-with-galera.md @@ -37,7 +37,7 @@ Install the [MariaDB repo](https://downloads.mariadb.org/mariadb/repositories/#d ```bash sudo apt-get install software-properties-common sudo apt-key adv --fetch-keys 'https://mariadb.org/mariadb_release_signing_key.asc' -sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://ftp.hosteurope.de/mirror/mariadb.org/repo/10.4/ubuntu bionic main' +sudo add-apt-repository 'deb [arch=amd64] http://ftp.hosteurope.de/mirror/mariadb.org/repo/10.4/ubuntu bionic main' sudo apt update sudo apt install mariadb-server mariadb-client mariadb-backup @@ -192,6 +192,138 @@ One the other 2 nodes run: `sudo systemctl start mariadb.service` mysql -u root -p -e "SELECT * FROM information_schema.global_status WHERE variable_name IN ('WSREP_CLUSTER_STATUS','WSREP_LOCAL_STATE_COMMENT','WSREP_CLUSTER_SIZE','WSREP_EVS_REPL_LATENCY','WSREP_EVS_DELAYED','WSREP_READY');" ``` + +##### Install and configure MaxScale + +```bash +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 135659E928C12247 +sudo add-apt-repository 'deb [arch=amd64] http://downloads.mariadb.com/MaxScale/2.2/ubuntu bionic main' +sudo apt install maxscale +``` + +``` +mysql -u root -p +CREATE USER 'maxscale'@'%' IDENTIFIED BY 'SuperSecretPassword'; +GRANT SELECT ON mysql.user TO 'maxscale'@'%'; +GRANT SELECT ON mysql.db TO 'maxscale'@'%'; +GRANT SELECT ON mysql.tables_priv TO 'maxscale'@'%'; +GRANT SELECT ON mysql.roles_mapping TO 'maxscale'@'%'; +GRANT SHOW DATABASES ON *.* TO 'maxscale'@'%'; +GRANT REPLICATION CLIENT ON *.* TO 'maxscale'@'%'; +GRANT SUPER ON *.* TO maxscale@'%'; +FLUSH PRIVILEGES; +exit +``` + +Generate MaxScale Keys **note the encrypted password** and write replace it in /etc/maxscale.cnf + +```bash +sudo systemctl stop maxscale +sudo maxkeys /var/lib/maxscale/ +sudo maxpasswd 'SuperSecretPassword' +sudo chown maxscale: /var/lib/maxscale/.secrets +``` + +/etc/maxscale.cnf + +```conf +# MaxScale documentation: +# https://mariadb.com/kb/en/mariadb-enterprise/mariadb-maxscale-22/ + +# Global parameters +# +# Complete list of configuration options: +# https://mariadb.com/kb/en/mariadb-enterprise/mariadb-maxscale-22-mariadb-maxscale-configuration-usage-scenarios/ + +[maxscale] +threads=auto + +# Server definitions +# +# Set the address of the server to the network +# address of a MariaDB server. +# + +[server1] +type=server +address=10.248.8.1 +port=3306 +protocol=MariaDBBackend + +[server2] +type=server +address=10.248.8.2 +port=3306 +protocol=MariaDBBackend + +[server3] +type=server +address=10.248.8.3 +port=3306 +protocol=MariaDBBackend + +# Monitor for the servers +# +# This will keep MaxScale aware of the state of the servers. +# MariaDB Monitor documentation: +# https://mariadb.com/kb/en/mariadb-enterprise/mariadb-maxscale-22-mariadb-monitor/ + +[MariaDB-Monitor] +type=monitor +module=galeramon +servers=server1,server2,server3 +user=maxscale +passwd=D83ED4E84351BD822950FDE5C2991889 +monitor_interval=2000 + +# Service definitions +# +# Service Definition for a read-only service and +# a read/write splitting service. +# + +# ReadWriteSplit documentation: +# https://mariadb.com/kb/en/mariadb-enterprise/mariadb-maxscale-22-readwritesplit/ + +[Read-Write-Service] +type=service +router=readwritesplit +servers=server1,server2,server3 +user=maxscale +passwd=D83ED4E84351BD822950FDE5C2991889 + +# This service enables the use of the MaxAdmin interface +# MaxScale administration guide: +# https://mariadb.com/kb/en/mariadb-enterprise/mariadb-maxscale-22-maxadmin-admin-interface/ + +[MaxAdmin-Service] +type=service +router=cli + +# Listener definitions for the services +# +# These listeners represent the ports the +# services will listen on. +# + +[Read-Write-Listener] +type=listener +service=Read-Write-Service +protocol=MariaDBClient +port=4006 + +[MaxAdmin-Listener] +type=listener +service=MaxAdmin-Service +protocol=maxscaled +socket=default +``` + +```bash +sudo systemctl start maxscale +sudo systemctl enable maxscale +``` + ##### Create the k3s Database One one node run: @@ -211,7 +343,7 @@ GRANT ALL PRIVILEGES ON `k3s`.* TO 'k3s'@'%' IDENTIFIED BY ''; Install k3s one each nodes, one after another: ```bash -curl -sfL https://get.k3s.io | sh -s - server --datastore-endpoint="mysql://k3s:@tcp(localhost:3306)/k3s" --no-deploy servicelb --no-deploy traefik +curl -sfL https://get.k3s.io | sh -s - server --datastore-endpoint="mysql://k3s:SuperSecretPassword@tcp(localhost:4006)/k3s" --no-deploy servicelb --no-deploy traefik ``` Check the nodes after. @@ -243,51 +375,58 @@ sudo apt-get install -y kubectl Install helm to ~/bin ```bash -wget https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz -tar xfz helm-v3.2.1-linux-amd64.tar.gz -mkdir ~/bin -mv linux-amd64/helm linux-amd64/tiller ~/bin -rm -rf linux-amd64 -``` - -Init helm on the cluster - -```bash -kubectl --namespace kube-system create serviceaccount tiller; -kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller; -helm init --service-account tiller; +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh ``` #### Install MetalLB Install MetalLB (change the address range!) +See: [metallb install](https://metallb.universe.tf/installation/) + ```bash -helm install stable/metallb \ - --name metallb \ - --namespace kube-system \ - --set configInline.address-pools[0].name=default \ - --set configInline.address-pools[0].protocol=layer2 \ - --set configInline.address-pools[0].addresses[0]=10.128.3.1-10.128.3.254 +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.3/manifests/namespace.yaml +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.3/manifests/metallb.yaml +# On first install only +kubectl create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" +``` + +metallb.yaml -> kubectl apply -f metallb.yaml + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - 10.248.11.1-10.248.11.253 ``` Check the deployment ```bash -kubectl get pods -n kube-system -l app=metallb -o wide +kubectl get pods -n metallb-system -l app=metallb -o wide ``` #### Install cert-manager for Let's Encrypt ```bash -kubectl apply -f https://raw.githubusercontent.com/jetstack/cert-manager/release-0.12/deploy/manifests/00-crds.yaml kubectl create namespace cert-manager +kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.0/cert-manager.crds.yaml helm repo add jetstack https://charts.jetstack.io helm repo update -helm install jetstack/cert-manager \ - --name cert-manager \ +helm install \ + cert-manager jetstack/cert-manager \ --namespace cert-manager \ - --version v0.12.0 + --version v0.15.0 ``` ```bash @@ -301,7 +440,10 @@ cert-manager-webhook-547567b88f-ptrlg 1/1 Running 0 54s #### Install Nginx ```bash -helm install nginx-ingress stable/nginx-ingress --namespace kube-system \ +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm install ingress-nginx ingress-nginx/ingress-nginx \ + --namespace kube-system \ --set controller.image.runAsUser=101 \ --set defaultBackend.enabled=false ``` @@ -309,10 +451,10 @@ helm install nginx-ingress stable/nginx-ingress --namespace kube-system \ #### Install Rancher ```bash -helm repo add rancher-stable https://releases.rancher.com/server-charts/stable +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +helm repo update kubectl create namespace cattle-system -helm install rancher-stable/rancher \ - --name rancher \ +helm install rancher rancher-latest/rancher \ --namespace cattle-system \ --set hostname=rancher.example.org \ --set ingress.tls.source=letsEncrypt \