From 78c5ba76e591f49b8b66e833c66577f18e9a792b Mon Sep 17 00:00:00 2001
From: Sebastiaan van Steenis <mail@superseb.nl>
Date: Sun, 1 Jul 2018 16:55:27 +0200
Subject: [PATCH] Add generic troubleshooting on network

---
 .../generic-troubleshooting/_index.md         | 87 ++++++++++++++++++-
 1 file changed, 83 insertions(+), 4 deletions(-)

diff --git a/content/rancher/v2.x/en/installation/troubleshooting-ha/generic-troubleshooting/_index.md b/content/rancher/v2.x/en/installation/troubleshooting-ha/generic-troubleshooting/_index.md
index ad66ce274b0..d1b696dfe42 100644
--- a/content/rancher/v2.x/en/installation/troubleshooting-ha/generic-troubleshooting/_index.md
+++ b/content/rancher/v2.x/en/installation/troubleshooting-ha/generic-troubleshooting/_index.md
@@ -5,6 +5,10 @@ weight: 5
 
 Below are steps that you can follow to determine what is wrong in your cluster.
 
+* Double check if all the required ports are opened in your (host) firewall
+
+Double check if all the [required ports]({{< baseurl >}}/rancher/v2.x/en/installation/references/) are opened in your (host) firewall.
+
 * All nodes should be present and in **Ready** state
 
 To check, run the command:
@@ -25,13 +29,13 @@ kubectl --kubeconfig kube_config_rancher-cluster.yml get pods --all-namespaces
 
 If a pod is not in **Running** state, you can dig into the root cause by running:
 
-<h5>Describe pod</h5>
+<h3>Describe pod</h3>
 
 ```
 kubectl --kubeconfig kube_config_rancher-cluster.yml describe pod POD_NAME -n NAMESPACE
 ```
 
-<h5>Pod container logs</h5>
+<h3>Pod container logs</h3>
 
 ```
 kubectl --kubeconfig kube_config_rancher-cluster.yml logs POD_NAME -n NAMESPACE
@@ -39,13 +43,13 @@ kubectl --kubeconfig kube_config_rancher-cluster.yml logs POD_NAME -n NAMESPACE
 
 If a job is not in **Completed** state, you can dig into the root cause by running:
 
-<h5>Describe job</h5>
+<h3>Describe job</h3>
 
 ```
 kubectl --kubeconfig kube_config_rancher-cluster.yml describe job JOB_NAME -n NAMESPACE
 ```
 
-<h5>Logs from the containers of pods of the job</h5>
+<h3>Logs from the containers of pods of the job</h3>
 
 ```
 kubectl --kubeconfig kube_config_rancher-cluster.yml logs -l job-name=JOB_NAME -n NAMESPACE
@@ -64,3 +68,78 @@ kubectl --kubeconfig kube_config_rancher-cluster.yml get events --all-namespaces
 ```
 kubectl --kubeconfig kube_config_rancher-cluster.yml logs -l app=cattle -n cattle-system
 ```
+
+* Check NGINX ingress controller logging
+
+```
+kubectl --kubeconfig kube_config_rancher-cluster.yml logs -l app=ingress-nginx -n ingress-nginx
+```
+
+* Check if overlay network is functioning correctly
+
+The pod can be scheduled to any of the hosts you used for your cluster, but that means that the NGINX ingress controller needs to be able to route the request from `NODE_1` to `NODE_2`. This happens over the overlay network. If the overlay network is not functioning, you will experience intermittent TCP/HTTP connection failures due to the NGINX ingress controller not being able to route to the pod.
+
+To test the overlay network, you can launch the following `DaemonSet` definition. This will run an `alpine` container on every host, which we will use to run a `ping` test between containers on all hosts.
+
+1. Save the following file as `ds-alpine.yml`
+
+    ```
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: alpine
+    spec:
+      selector:
+          matchLabels:
+            name: alpine
+      template:
+        metadata:
+          labels:
+            name: alpine
+        spec:
+          tolerations:
+          - effect: NoExecute
+            key: "node-role.kubernetes.io/etcd"
+            value: "true"
+          - effect: NoSchedule
+            key: "node-role.kubernetes.io/controlplane"
+            value: "true"
+          containers:
+          - image: alpine
+            imagePullPolicy: Always
+            name: alpine
+            command: ["sh", "-c", "tail -f /dev/null"]
+            terminationMessagePath: /dev/termination-log
+    ```
+
+2. Launch it using `kubectl --kubeconfig kube_config_rancher-cluster.yml create -f ds-alpine.yml`
+3. Wait until `kubectl --kubeconfig kube_config_rancher-cluster.yml rollout status ds/alpine -w` returns: `daemon set "alpine" successfully rolled out`.
+4. Run the following command to let each container on every host ping each other (it's a single line command).
+
+    ```
+    echo "=> Start"; kubectl --kubeconfig kube_config_rancher-cluster.yml get pods -l name=alpine -o jsonpath='{range .items[*]}{@.metadata.name}{" "}{@.spec.nodeName}{"\n"}{end}' | while read spod shost; do kubectl --kubeconfig kube_config_rancher-cluster.yml get pods -l name=alpine -o jsonpath='{range .items[*]}{@.status.podIP}{" "}{@.spec.nodeName}{"\n"}{end}' | while read tip thost; do kubectl --kubeconfig kube_config_rancher-cluster.yml --request-timeout='10s' exec $spod -- /bin/sh -c "ping -c2 $tip > /dev/null 2>&1"; RC=$?; if [ $RC -ne 0 ]; then echo $shost cannot reach $thost; fi; done; done; echo "=> End"
+    ```
+
+5. When this command has finished running, the output indicating everything is correct is:
+
+    ```
+    => Start
+    => End
+    ```
+
+If you see error in the output, that means that the [required ports]({{< baseurl >}}/rancher/v2.x/en/installation/references/) for overlay networking are not opened between the hosts indicated.
+
+Example error output of a situation where NODE1 had the UDP ports blocked.
+
+```
+=> Start
+command terminated with exit code 1
+NODE2 cannot reach NODE1
+command terminated with exit code 1
+NODE3 cannot reach NODE1
+command terminated with exit code 1
+NODE1 cannot reach NODE2
+command terminated with exit code 1
+NODE1 cannot reach NODE3
+=> End
+```