diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e713d4aacde..b80ff521263 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,7 +25,7 @@ jobs: run: yarn install --frozen-lockfile - name: Build website env: - NODE_OPTIONS: "--max_old_space_size=7168" + NODE_OPTIONS: "--max_old_space_size=8192" run: yarn build --no-minify - name: Upload Build Artifact diff --git a/README.md b/README.md index e91e393074b..f0eba103494 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ To get started, [fork](https://github.com/rancher/rancher-docs/fork) and clone t Our repository doesn't allow you to make changes directly to the `main` branch. Create a working branch and make pull requests from your fork to [rancher/rancher-docs](https://github.com/rancher/rancher-docs). -For most updates, you'll need to edit a file in the `/docs` directory, which represents the ["Latest"](https://ranchermanager.docs.rancher.com/) version of our published documentation. The "Latest" version is a mirror of the most recently released version of Rancher. As of August 2024, the most recently released version of Rancher is 2.9. +For most updates, you'll need to edit a file in the `/docs` directory, which represents the ["Latest"](https://ranchermanager.docs.rancher.com/) version of our published documentation. The "Latest" version is a mirror of the most recently released version of Rancher. As of July 2025, the most recently released version of Rancher is 2.12. Whenever an update is made to `/docs`, you should apply the same change to the corresponding file in `/versioned_docs/version-2.9`. If a change only affects older versions, you don't need to mirror it to the `/docs` directory. diff --git a/docs/api/api-reference.mdx b/docs/api/api-reference.mdx index c0eedae1159..ca78ba81cc8 100644 --- a/docs/api/api-reference.mdx +++ b/docs/api/api-reference.mdx @@ -15,4 +15,4 @@ At this time, not all Rancher resources are available through the Rancher Kubern import ApiDocMdx from '@theme/ApiDocMdx'; - + diff --git a/docs/api/extension-apiserver.md b/docs/api/extension-apiserver.md index b80de8af261..fa4556046b9 100644 --- a/docs/api/extension-apiserver.md +++ b/docs/api/extension-apiserver.md @@ -16,4 +16,4 @@ All versions of Kubernetes supported by Rancher with the feature will have the a :::note If the underlying Kubernetes distribution does not support the aggregation layer, you must migrate to a Kubernetes distribution that does before upgrading. -::: \ No newline at end of file +::: diff --git a/docs/api/v3-rancher-api-guide.md b/docs/api/v3-rancher-api-guide.md index 14bc92a5bed..e04b0896c0e 100644 --- a/docs/api/v3-rancher-api-guide.md +++ b/docs/api/v3-rancher-api-guide.md @@ -72,7 +72,7 @@ API responses are paginated with a limit of 100 resources per page by default. T ## Capturing v3 API Calls -You can use browser developer tools to capture how the v3 API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning an RKE cluster: +You can use browser developer tools to capture how the v3 API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning a Rancher Kubernetes distribution cluster: 1. In the Rancher UI, go to **Cluster Management** and click **Create.** 1. Click one of the cluster types. This example uses Digital Ocean. diff --git a/docs/api/workflows/kubeconfigs.md b/docs/api/workflows/kubeconfigs.md new file mode 100644 index 00000000000..a6e156f3272 --- /dev/null +++ b/docs/api/workflows/kubeconfigs.md @@ -0,0 +1,205 @@ +--- +title: Kubeconfigs +--- + + + + + +## Kubeconfig Resource + +Kubeconfig is a Rancher resource `kubeconfigs.ext.cattle.io` that allows generating `v1.Config` kubeconfig files for interacting with Rancher and clusters managed by Rancher. + +```sh +kubectl api-resources --api-group=ext.cattle.io +``` + +To get a description of the fields and structure of the Kubeconfig resource, run: + +```sh +kubectl explain kubeconfigs.ext.cattle.io +``` + +## Feature Flag + +The Kubeconfigs Public API is available since Rancher v2.12.0 and is enabled by default. It can be disabled by setting the `ext-kubeconfigs` feature flag to `false`. + +```sh +kubectl patch feature ext-kubeconfigs -p '{"spec":{"value":false}}' +``` + +## Creating a Kubeconfig + +Only a **valid and active** Rancher user can create a Kubeconfig. For example, trying to create a Kubeconfig using a `system:admin` service account will lead to an error: + +```bash +kubectl create -o jsonpath='{.status.value}' -f -< + + + +## Token Resource + +Rancher has an imperative API resource `tokens.ext.cattle.io` that allows you to generate tokens for authenticating with Rancher. + +```sh +kubectl api-resources --api-group=ext.cattle.io +``` + +To get a description of the fields and structure of the Token resource, run: + +```sh +kubectl explain tokens.ext.cattle.io +``` + +## Feature Flag + +The Tokens Public API is available for Rancher v2.12.0 and later, and is enabled by default. You can disable the Tokens Public API by setting the `ext-tokens` feature flag to `false` as shown in the example `kubectl` command below: + +```sh +kubectl patch feature ext-tokens -p '{"spec":{"value":false}}' +``` + +## Creating a Token + +:::caution +The Token value is only returned once in the `status.value` field. +::: + +Only a **valid and active** Rancher user can create a Token. Otherwise, you will get an error displayed (`Error from server (Forbidden)...`) when attempting to create a Token. + +```bash +kubectl create -o jsonpath='{.status.value}' -f -< - -![Weave Logo](/img/weave-logo.png) - -Weave enables networking and network policy in Kubernetes clusters across the cloud. Additionally, it support encrypting traffic between the peers. - -Kubernetes workers should open TCP port `6783` (control port), UDP port `6783` and UDP port `6784` (data ports). See the [port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. - -For more information, see the following pages: - -- [Weave Net Official Site](https://github.com/weaveworks/weave/blob/master/site/overview.md) - ### RKE2 Kubernetes clusters -Out-of-the-box, Rancher provides the following CNI network providers for RKE2 Kubernetes clusters: [Canal](#canal) (see above section), Calico, and Cilium. +Out-of-the-box, Rancher provides the following CNI network providers for RKE2 Kubernetes clusters: Calico, Canal, Cilium, and Flannel. You can choose your CNI network provider when you create new Kubernetes clusters from Rancher. @@ -131,6 +80,20 @@ For more information, see the following pages: - [Project Calico Official Site](https://www.projectcalico.org/) - [Project Calico GitHub Page](https://github.com/projectcalico/calico) +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal is a CNI network provider that gives you the best of Flannel and Calico. It allows users to easily deploy Calico and Flannel networking together as a unified networking solution, combining Calico’s network policy enforcement with the rich superset of Calico (unencapsulated) and/or Flannel (encapsulated) network connectivity options. + +In Rancher, Canal is the default CNI network provider combined with Flannel and VXLAN encapsulation. + +Kubernetes workers should open UDP port `8472` (VXLAN) and TCP port `9099` (health checks). If using Wireguard, you should open UDP ports `51820` and `51821`. For more details, refer to [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md). + +![](/img/canal-diagram.png) + +For more information, see the [Canal GitHub Page.](https://github.com/projectcalico/canal) + #### Cilium ![Cilium Logo](/img/cilium-logo.png) @@ -158,6 +121,23 @@ spec: - remote-node ``` +#### Flannel + +![Flannel Logo](/img/flannel-logo.png) + +Flannel is a simple and easy way to configure L3 network fabric designed for Kubernetes. Flannel runs a single binary agent named flanneld on each host, which is responsible for allocating a subnet lease to each host out of a larger, preconfigured address space. Flannel uses either the Kubernetes API or etcd directly to store the network configuration, the allocated subnets, and any auxiliary data (such as the host's public IP). Packets are forwarded using one of several backend mechanisms, with the default encapsulation being [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan). + +Encapsulated traffic is unencrypted by default. Flannel provides two solutions for encryption: + +* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec), which makes use of [strongSwan](https://www.strongswan.org/) to establish encrypted IPSec tunnels between Kubernetes workers. It is an experimental backend for encryption. +* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard), which is a more faster-performing alternative to strongSwan. + +Kubernetes workers should open UDP port `8472` (VXLAN). See [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +![Flannel Diagram](/img/flannel-diagram.png) + +For more information, see the [Flannel GitHub Page](https://github.com/flannel-io/flannel). + ## CNI Features by Provider The following table summarizes the different features available for each CNI network provider provided by Rancher. @@ -196,4 +176,4 @@ Canal is the default CNI network provider. We recommend it for most use cases. I ## How can I configure a CNI network provider? -Please see [Cluster Options](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) on how to configure a network provider for your cluster. For more advanced configuration options, please see how to configure your cluster using a [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) and the options for [Network Plug-ins](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/). +Please see [Cluster Options](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) on how to configure a network provider for your cluster. For more advanced configuration options, please see how to configure your cluster using a [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#cluster-config-file-reference). diff --git a/docs/faq/deprecated-features.md b/docs/faq/deprecated-features.md index 959e3edf1d2..617d6e5f6bc 100644 --- a/docs/faq/deprecated-features.md +++ b/docs/faq/deprecated-features.md @@ -16,10 +16,7 @@ Rancher will publish deprecated features as part of the [release notes](https:// | Patch Version | Release Date | |---------------|---------------| -| [2.11.3](https://github.com/rancher/rancher/releases/tag/v2.11.3) | June 25, 2025 | -| [2.11.2](https://github.com/rancher/rancher/releases/tag/v2.11.2) | May 22, 2025 | -| [2.11.1](https://github.com/rancher/rancher/releases/tag/v2.11.1) | Apr 24, 2025 | -| [2.11.0](https://github.com/rancher/rancher/releases/tag/v2.11.0) | Mar 31, 2025 | +| [2.12.0](https://github.com/rancher/rancher/releases/tag/v2.12.0) | July 30, 2025 | ## What can I expect when a feature is marked for deprecation? diff --git a/docs/faq/dockershim.md b/docs/faq/dockershim.md deleted file mode 100644 index 387b20b3eee..00000000000 --- a/docs/faq/dockershim.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Dockershim FAQ ---- - - - - - -The Dockershim is the CRI compliant layer between the Kubelet and the Docker daemon. As part of the Kubernetes 1.20 release, the [deprecation of the in-tree Dockershim was announced](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/). Removal is currently scheduled for Kubernetes 1.24. For more information on the deprecation and its timelines, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). - -RKE clusters, starting with Kubernetes 1.21, now support the external Dockershim to continue leveraging Docker as the CRI runtime. We now implement the upstream open source community Dockershim announced by [Mirantis and Docker](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) to ensure RKE clusters can continue to leverage Docker. - -To enable the external Dockershim, configure the following option. - -``` -enable_cri_dockerd: true -``` - -For users looking to use another container runtime, Rancher has the edge-focused K3s and datacenter-focused RKE2 Kubernetes distributions that use containerd as the default runtime. Imported RKE2 and K3s Kubernetes clusters can then be upgraded and managed through Rancher even after the removal of in-tree Dockershim in Kubernetes 1.24. - -## FAQ - -
- -Q: Do I have to upgrade Rancher to get Rancher’s support of the upstream Dockershim? - -The upstream support of Dockershim begins for RKE in Kubernetes 1.21. You will need to be on Rancher 2.6 or above to have support for RKE with Kubernetes 1.21. See our [support matrix](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/) for details. - -
- -Q: I am currently on RKE with Kubernetes 1.20. Do I need to upgrade to RKE with Kubernetes 1.21 sooner to avoid being out of support for Dockershim? - -A: The version of Dockershim in RKE with Kubernetes 1.20 will continue to work and is not scheduled for removal upstream until Kubernetes 1.24. It will only emit a warning of its future deprecation, which Rancher has mitigated in RKE with Kubernetes 1.21. You can plan your upgrade to Kubernetes 1.21 as you would normally, but should consider enabling the external Dockershim by Kubernetes 1.22. The external Dockershim will need to be enabled before upgrading to Kubernetes 1.24, at which point the existing implementation will be removed. - -For more information on the deprecation and its timeline, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). - -
- -Q: What are my other options if I don’t want to depend on the Dockershim? - -A: You can use a runtime like containerd with Kubernetes that does not require Dockershim support. RKE2 or K3s are two options for doing this. - -
- -Q: If I am already using RKE1 and want to switch to RKE2, what are my migration options? - -A: Rancher is exploring the possibility of an in-place upgrade path. Alternatively you can always migrate workloads from one cluster to another using kubectl. - -
diff --git a/docs/faq/install-and-configure-kubectl.md b/docs/faq/install-and-configure-kubectl.md index 868eb42caac..3022cae6816 100644 --- a/docs/faq/install-and-configure-kubectl.md +++ b/docs/faq/install-and-configure-kubectl.md @@ -14,17 +14,11 @@ See [kubectl Installation](https://kubernetes.io/docs/tasks/tools/install-kubect ## Configuration -When you create a Kubernetes cluster with RKE, RKE creates a `kube_config_cluster.yml` in the local directory that contains credentials to connect to your new cluster with tools like `kubectl` or `helm`. - -You can copy this file as `$HOME/.kube/config` or if you are working with multiple Kubernetes clusters, set the `KUBECONFIG` environmental variable to the path of `kube_config_cluster.yml`. - -``` -export KUBECONFIG=$(pwd)/kube_config_cluster.yml -``` +When you create a Kubernetes cluster with RKE2/K3s, the Kubeconfig file is stored at `/etc/rancher/rke2/rke2.yaml` or `/etc/rancher/k3s/k3s.yaml` depending on your chosen distribution. These files are used to configure access to the Kubernetes cluster. Test your connectivity with `kubectl` and see if you can get the list of nodes back. -``` +```shell kubectl get nodes NAME STATUS ROLES AGE VERSION 165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 diff --git a/docs/faq/rancher-is-no-longer-needed.md b/docs/faq/rancher-is-no-longer-needed.md index 1752bd29b37..e1d1f759206 100644 --- a/docs/faq/rancher-is-no-longer-needed.md +++ b/docs/faq/rancher-is-no-longer-needed.md @@ -17,9 +17,9 @@ If Rancher is ever deleted or unrecoverable, all workloads in the downstream Kub The capability to access a downstream cluster without Rancher depends on the type of cluster and the way that the cluster was created. To summarize: -- **Registered clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. +- **Registered/Imported clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. - **Hosted Kubernetes clusters:** If you created the cluster in a cloud-hosted Kubernetes provider such as EKS, GKE, or AKS, you can continue to manage the cluster using your provider's cloud credentials. -- **RKE clusters:** To access an [RKE cluster,](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. (The authorized cluster endpoint is enabled by default for RKE clusters.) With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. +- **Rancher provisioned clusters:** To access an [RKE2/K3s cluster](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. ## What if I don't want Rancher anymore? @@ -56,10 +56,10 @@ To detach the cluster, **Result:** The registered cluster is detached from Rancher and functions normally outside of Rancher. -## What if I don't want my RKE cluster or hosted Kubernetes cluster managed by Rancher? +## What if I don't want my hosted Kubernetes cluster managed by Rancher? At this time, there is no functionality to detach these clusters from Rancher. In this context, "detach" is defined as the ability to remove Rancher components from the cluster and manage access to the cluster independently of Rancher. The capability to manage these clusters without Rancher is being tracked in this [issue.](https://github.com/rancher/rancher/issues/25234) -For information about how to access clusters if the Rancher server is deleted, refer to [this section.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) +For information about how to access clusters if the Rancher server is deleted, refer to [this section.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) \ No newline at end of file diff --git a/docs/faq/technical-items.md b/docs/faq/technical-items.md index 985cf6a5dbf..cdf29707617 100644 --- a/docs/faq/technical-items.md +++ b/docs/faq/technical-items.md @@ -64,7 +64,7 @@ The Layer-4 Load Balancer is created as `type: LoadBalancer`. In Kubernetes, thi ## Where is the state of Rancher stored? - Docker Install: in the embedded etcd of the `rancher/rancher` container, located at `/var/lib/rancher`. -- Kubernetes install: in the etcd of the RKE cluster created to run Rancher. +- Kubernetes install: default location is in the `/var/lib/rancher/rke2` or `/var/lib/rancher/k3s` directories of the respective RKE2/K3s cluster created to run Rancher. ## How are the supported Docker versions determined? @@ -99,7 +99,7 @@ When the node is removed from the cluster, and the node is cleaned, you can add ## How can I add more arguments/binds/environment variables to Kubernetes components in a Rancher Launched Kubernetes cluster? -You can add more arguments/binds/environment variables via the [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) option in Cluster Options. For more information, see the [Extra Args, Extra Binds, and Extra Environment Variables](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/) in the RKE documentation or browse the [Example Cluster.ymls](https://rancher.com/docs/rke/latest/en/example-yamls/). +You can add more arguments/binds/environment variables via the respective [RKE2 Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#cluster-configuration) or [K3s Config File](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md#cluster-configuration). ## How do I check if my certificate chain is valid? diff --git a/docs/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/docs/getting-started/installation-and-upgrade/installation-references/feature-flags.md index ce153bce599..e8ef27b5c19 100644 --- a/docs/getting-started/installation-and-upgrade/installation-references/feature-flags.md +++ b/docs/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -26,32 +26,54 @@ The following is a list of feature flags available in Rancher. If you've upgrade - `imperative-api-extension`: Enables Rancher's [extension API server](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) to register new APIs to Kubernetes. This flag is enabled by default. See the [Extension API Server](../../../api/extension-apiserver.md) page for more information. - `istio-virtual-service-ui`: Enables a [visual interface](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md) to create, read, update, and delete Istio virtual services and destination rules, which are Istio traffic management features. - `legacy`: Enables a set of features from 2.5.x and earlier, that are slowly being phased out in favor of newer implementations. These are a mix of deprecated features as well as features that will eventually be available to newer versions. This flag is disabled by default on new Rancher installations. If you're upgrading from a previous version of Rancher, this flag is enabled. -- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream RKE2/K3s clusters, currently limited to imported clusters and the local cluster, with plans to expand support to node-driver clusters. +- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream imported RKE2/K3s clusters, as well as in the local cluster if it is an RKE2/K3s cluster. + +:::note Important: + +This `managed-system-upgrade-controller` flag is intended for **internal use only** and does not have an associated Feature CR. Use with caution. + +To control whether Rancher should manage the Kubernetes version of imported RKE2/K3s clusters, it is recommended to use the [imported-cluster-version-management](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters) feature that is available in Rancher v2.11.0 or newer. + +::: + +:::danger + +If the `managed-system-upgrade-controller` flag was **disabled** in Rancher v2.10.x, and any imported RKE2/K3s clusters were upgraded **outside of Rancher**, follow the steps below to prevent the unexpected installation of the system-upgrade-controller app and to ensure the [imported-cluster-version-management](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters) feature works correctly: + +1. Upgrade Rancher to v2.11.0 or newer, making sure to **retain** the `managed-system-upgrade-controller=false` feature flag in Helm values if it was set during the v2.10.x installation. +1. After Rancher is fully up and running, disable the `imported-cluster-version-management` setting. You can do this either through the Rancher UI by clicking **☰ > Global Settings > Settings > imported-cluster-version-management**, or by editing the corresponding `Setting.management.cattle.io/v3` custom resource via kubectl. +1. Perform a second Helm upgrade, this time omitting the `managed-system-upgrade-controller=false` feature flag. + +Now, the imported cluster version management is disabled by default, and Rancher no longer installs the system-upgrade-controller app on imported clusters automatically. + +You can enable this feature on a per-cluster basis. For more information, please refer to the [documentation](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters). + +::: + - `multi-cluster-management`: Allows multi-cluster provisioning and management of Kubernetes clusters. This flag can only be set at install time. It can't be enabled or disabled later. - `rke1-custom-node-cleanup`: Enables cleanup of deleted RKE1 custom nodes. We recommend that you keep this flag enabled, to prevent removed nodes from attempting to rejoin the cluster. - `rke2`: Enables provisioning RKE2 clusters. This flag is enabled by default. - `token-hashing`: Enables token hashing. Once enabled, existing tokens will be hashed and all new tokens will be hashed automatically with the SHA256 algorithm. Once a token is hashed it can't be undone. This flag can't be disabled after its enabled. See [API Tokens](../../../api/api-tokens.md#token-hashing) for more information. -- `uiextension`: Enables UI extensions. This flag is enabled by default. Enabling or disabling the flag forces the Rancher pod to restart. The first time this flag is set to `true`, it creates a CRD and enables the controllers and endpoints necessary for the feature to work. If set to `false`, it disables the previously mentioned controllers and endpoints. Setting `uiextension` to `false` has no effect on the CRD -- it does not create a CRD if it does not yet exist, nor does it delete the CRD if it already exists. +- `uiextension`: Enables UI extensions. This flag is enabled by default. Enabling or disabling the flag forces the Rancher pod to restart. The first time this flag is set to `Active`, it creates a CRD and enables the controllers and endpoints necessary for the feature to work. If set to `Disabled`, it disables the previously mentioned controllers and endpoints. Setting `uiextension` to `Disabled` has no effect on the CRD -- it does not create a CRD if it does not yet exist, nor does it delete the CRD if it already exists. - `unsupported-storage-drivers`: Enables types for storage providers and provisioners that aren't enabled by default. See [Allow Unsupported Storage Drivers](../../../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) for more information. -- `ui-sql-cache`: Enables a SQLite-based cache for UI tables. See [UI Server-Side Pagination](../../../how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md) for more information. - +- `ui-sql-cache`: Enables an SQLite-based cache for UI tables and Server-Side Pagination. See [UI Server-Side Pagination](../../../how-to-guides/advanced-user-guides/ui-server-side-pagination.md) for more information. The following table shows the availability and default values for some feature flags in Rancher. Features marked "GA" are generally available: | Feature Flag Name | Default Value | Status | Available As Of | Additional Information | | ----------------------------- | ------------- | ------------ | --------------- | ---------------------- | -| `aggregated-roletemplates` | `false` | Highly experimentatl | v2.11.0 | This flag value is locked on install and can't be changed. | -| `clean-stale-secrets` | `true` | GA | v2.10.2 | | -| `continuous-delivery` | `true` | GA | v2.6.0 | | -| `external-rules` | v2.7.14: `false`, v2.8.5: `true` | Removed | v2.7.14, v2.8.5 | This flag affected [external `RoleTemplate` behavior](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#external-roletemplate-behavior). It is removed in Rancher v2.9.0 and later as the behavior is enabled by default. | -| `fleet` | `true` | Can no longer be disabled | v2.6.0 | | -| `fleet` | `true` | GA | v2.5.0 | | -| `harvester` | `true` | Experimental | v2.6.1 | | -| `imperative-api-extension` | `true` | GA | v2.11.0 | | -| `legacy` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | | -| `managed-system-upgrade-controller` | `true` | GA | v2.10.0 | | +| `aggregated-roletemplates` | `Disabled` | Highly experimental | v2.11.0 | This flag value is locked on install and can't be changed. | +| `clean-stale-secrets` | `Active` | GA | v2.10.2 | | +| `continuous-delivery` | `Active` | GA | v2.6.0 | | +| `external-rules` | v2.7.14: `Disabled`, v2.8.5: `Active` | Removed | v2.7.14, v2.8.5 | This flag affected [external `RoleTemplate` behavior](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#external-roletemplate-behavior). It is removed in Rancher v2.9.0 and later as the behavior is enabled by default. | +| `fleet` | `Active` | Can no longer be disabled | v2.6.0 | | +| `fleet` | `Active` | GA | v2.5.0 | | +| `harvester` | `Active` | Experimental | v2.6.1 | | +| `imperative-api-extension` | `Active` | GA | v2.11.0 | | +| `legacy` | `Disabled` for new installs, `Active` for upgrades | GA | v2.6.0 | | +| `managed-system-upgrade-controller` | `Active` | GA | v2.10.0 | | | `rke1-custom-node-cleanup`| `true` | GA | v2.6.0 | | | `rke2` | `true` | Experimental | v2.6.0 | | -| `token-hashing` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | | -| `uiextension` | `true` | GA | v2.9.0 | | -| `ui-sql-cache` | `false` | Highly experimental | v2.9.0 | | +| `token-hashing` | `Disabled` for new installs, `Active` for upgrades | GA | v2.6.0 | | +| `uiextension` | `Active` | GA | v2.9.0 | | +| `ui-sql-cache` | `Active` | GA | v2.9.0 | | diff --git a/docs/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md b/docs/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md index aa8da6f793f..8dedc6288bf 100644 --- a/docs/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md +++ b/docs/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md @@ -36,7 +36,8 @@ For information on enabling experimental features, refer to [this page.](../../. | `antiAffinity` | "preferred" | `string` - AntiAffinity rule for Rancher pods - "preferred, required" | | `auditLog.destination` | "sidecar" | `string` - Stream to sidecar container console or hostPath volume - "sidecar, hostPath" | | `auditLog.hostPath` | "/var/log/rancher/audit" | `string` - log file destination on host (only applies when `auditLog.destination` is set to `hostPath`) | -| `auditLog.level` | 0 | `int` - set the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) level. 0 is off. [0-3] | +| `auditLog.enabled` | false | `bool` - Enables / disables audit logging. | +| `auditLog.level` | 0 | `int` - Sets the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) level [0-3]. | | `auditLog.maxAge` | 1 | `int` - maximum number of days to retain old audit log files (only applies when `auditLog.destination` is set to `hostPath`) | | `auditLog.maxBackup` | 1 | `int` - maximum number of audit log files to retain (only applies when `auditLog.destination` is set to `hostPath`) | | `auditLog.maxSize` | 100 | `int` - maximum size in megabytes of the audit log file before it gets rotated (only applies when `auditLog.destination` is set to `hostPath`) | @@ -63,6 +64,10 @@ For information on enabling experimental features, refer to [this page.](../../. | `tls` | "ingress" | `string` - See [External TLS Termination](#external-tls-termination) for details. - "ingress, external" | | `useBundledSystemChart` | `false` | `bool` - select to use the system-charts packaged with Rancher server. This option is used for air gapped installations. | + +When using Rancher v2.12.0 and above, Rancher will use an audit logging controller that watches `AuditPolicy` CRs for configuring additional redactions, for more info see [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md). + + ### Bootstrap Password You can [set a specific bootstrap password](../resources/bootstrap-password.md) during Rancher installation. If you don't set a specific bootstrap password, Rancher randomly generates a password for the first admin account. diff --git a/docs/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/docs/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md index c4618be475f..fbdc5c0356e 100644 --- a/docs/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md +++ b/docs/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -53,6 +53,10 @@ You can obtain `` and `` by loggi ## Upgrade +:::danger +Rancher upgrades to version 2.12.0 and later will be blocked if any RKE1-related resources are detected, as the Rancher Kubernetes Engine (RKE/RKE1) is end of life as of **July 31, 2025**. For detailed cleanup and recovery steps, refer to the [RKE1 Resource Validation and Upgrade Requirements in Rancher v2.12](#rke1-resource-validation-and-upgrade-requirements-in-rancher-v212). +::: + During upgrade, you create a copy of the data from your current Rancher container and a backup in case something goes wrong. Then you deploy the new version of Rancher in a new container using your existing data. ### 1. Create a copy of the data from your Rancher server container @@ -388,6 +392,45 @@ See [Restoring Cluster Networking](https://github.com/rancher/rancher-docs/tree/ Remove the previous Rancher server container. If you only stop the previous Rancher server container (and don't remove it), the container may restart after the next server reboot. +## RKE1 Resource Validation and Upgrade Requirements in Rancher v2.12 + +Rancher v2.12.0 and later has removed support for the Rancher Kubernetes Engine (RKE/RKE1). During upgrade, Rancher validates the cluster resources and blocks the upgrade if any RKE1-related resources are detected. + +This validation affects the following resource types: + +- Clusters with `rkeConfig` (`clusters.management.cattle.io`) +- NodeTemplates (`nodetemplates.management.cattle.io`) +- ClusterTemplates (`clustertemplates.management.cattle.io`) + +This is particularly relevant for single-node Docker installations, where Rancher is not running during the upgrade. In such cases, controllers are not available to automatically clean up deprecated resources, and the upgrade process will fail early with an error listing the blocking resources. + +### 1. Pre-Upgrade (Recommended) + +Before upgrading, while Rancher is still running: + +- Run the `pre-upgrade-hook` cleanup script to delete all RKE1 clusters and templates. You can find the script in the Rancher GitHub repository: [pre-upgrade-hook.sh](https://github.com/rancher/rancher/blob/v2.12.0/chart/scripts/pre-upgrade-hook.sh). +- This allows Rancher to clean up associated resources and finalizers. + +### 2. Post-Upgrade Failure Due to Residual RKE1 Resources + +If the upgrade to Rancher v2.12.0 or later is attempted without prior cleanup of RKE1 resources: + +- The upgrade will fail and display an error listing the resource names that are preventing the upgrade. +- This occurs because Rancher includes validation to detect and block upgrades when unsupported RKE1 resources are still present. +- To proceed, [rollback](#rolling-back) to the previous Rancher version, delete the identified resources, and then retry after [manual cleanup](#manual-cleanup-after-rollback). + +:::note Helm-based Rancher +Helm-based Rancher installations are not affected by this issue, as Rancher remains available during the upgrade and can perform resource cleanup as needed. +::: + +### Manual Cleanup After Rollback + +Users should perform the following steps after rolling back to a previous Rancher version: + +- **Manually delete** the resources listed in the upgrade error message (e.g., RKE1 clusters, NodeTemplates, ClusterTemplates). +- If deletion is blocked due to **finalizers**, edit the resources and remove the `metadata.finalizers` field. +- If a **validating webhook** prevents deletion (e.g., for the `system-project`), please refer to the [Bypassing the Webhook](../../../../reference-guides/rancher-webhook.md#bypassing-the-webhook) documentation. + ## Rolling Back If your upgrade does not complete successfully, you can roll back Rancher server and its data back to its last healthy state. For more information, see [Docker Rollback](roll-back-docker-installed-rancher.md). diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md deleted file mode 100644 index a7c6ed43472..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: CIS Scan Guides ---- - - - - - -- [Install rancher-cis-benchmark](install-rancher-cis-benchmark.md) -- [Uninstall rancher-cis-benchmark](uninstall-rancher-cis-benchmark.md) -- [Run a Scan](run-a-scan.md) -- [Run a Scan Periodically on a Schedule](run-a-scan-periodically-on-a-schedule.md) -- [Skip Tests](skip-tests.md) -- [View Reports](view-reports.md) -- [Enable Alerting for rancher-cis-benchmark](enable-alerting-for-rancher-cis-benchmark.md) -- [Configure Alerts for Periodic Scan on a Schedule](configure-alerts-for-periodic-scan-on-a-schedule.md) -- [Create a Custom Benchmark Version to Run](create-a-custom-benchmark-version-to-run.md) \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md deleted file mode 100644 index 8d3b66c7e4e..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Create a Custom Benchmark Version for Running a Cluster Scan ---- - - - - - -There could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. - -It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. - -For details, see [this page.](../../../integrations-in-rancher/cis-scans/custom-benchmark.md) \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md deleted file mode 100644 index ef2b5ae330d..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Enable Alerting for Rancher CIS Benchmark ---- - - - - - -Alerts can be configured to be sent out for a scan that runs on a schedule. - -:::note Prerequisite: - -Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) - -While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) - -::: - -While installing or upgrading the `rancher-cis-benchmark` Helm chart, set the following flag to `true` in the `values.yaml`: - -```yaml -alerts: - enabled: true -``` \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md deleted file mode 100644 index 2fede69bee6..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Run a Scan ---- - - - - - -When a ClusterScan custom resource is created, it launches a new CIS scan on the cluster for the chosen ClusterScanProfile. - -:::note - -There is currently a limitation of running only one CIS scan at a time for a cluster. If you create multiple ClusterScan custom resources, they will be run one after the other by the operator, and until one scan finishes, the rest of the ClusterScan custom resources will be in the "Pending" state. - -::: - -To run a scan, - -1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. -1. Click **Create**. -1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. -1. Click **Create**. - -**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md deleted file mode 100644 index 7492bc03f0b..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Skip Tests ---- - - - - - -CIS scans can be run using test profiles with user-defined skips. - -To skip tests, you will create a custom CIS scan profile. A profile contains the configuration for the CIS scan, which includes the benchmark versions to use and any specific tests to skip in that benchmark. - -1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Profile**. -1. From here, you can create a profile in multiple ways. To make a new profile, click **Create** and fill out the form in the UI. To make a new profile based on an existing profile, go to the existing profile and click **⋮ Clone**. If you are filling out the form, add the tests to skip using the test IDs, using the relevant CIS Benchmark as a reference. If you are creating the new test profile as YAML, you will add the IDs of the tests to skip in the `skipTests` directive. You will also give the profile a name: - - ```yaml - apiVersion: cis.cattle.io/v1 - kind: ClusterScanProfile - metadata: - annotations: - meta.helm.sh/release-name: clusterscan-operator - meta.helm.sh/release-namespace: cis-operator-system - labels: - app.kubernetes.io/managed-by: Helm - name: "" - spec: - benchmarkVersion: cis-1.5 - skipTests: - - "1.1.20" - - "1.1.21" - ``` -1. Click **Create**. - -**Result:** A new CIS scan profile is created. - -When you [run a scan](./run-a-scan.md) that uses this profile, the defined tests will be skipped during the scan. The skipped tests will be marked in the generated report as `Skip`. diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md deleted file mode 100644 index df23f7abbdc..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Uninstall Rancher CIS Benchmark ---- - - - - - -1. From the **Cluster Dashboard,** go to the left navigation bar and click **Apps > Installed Apps**. -1. Go to the `cis-operator-system` namespace and check the boxes next to `rancher-cis-benchmark-crd` and `rancher-cis-benchmark`. -1. Click **Delete** and confirm **Delete**. - -**Result:** The `rancher-cis-benchmark` application is uninstalled. \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md b/docs/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md deleted file mode 100644 index bb9045033bc..00000000000 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: View Reports ---- - - - - - -To view the generated CIS scan reports, - -1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. -1. The **Scans** page will show the generated reports. To see a detailed report, go to a scan report and click the name. - -One can download the report from the Scans list or from the scan detail page. - -To get the verbose version of the CIS scan results, run the following command on the cluster that was scanned. Note that the scan must be completed before this can be done. - -```console -export REPORT="scan-report-name" -kubectl get clusterscanreport $REPORT -o json |jq ".spec.reportJSON | fromjson" | jq -r ".actual_value_map_data" | base64 -d | gunzip | jq . -``` diff --git a/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md new file mode 100644 index 00000000000..c90922ec778 --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md @@ -0,0 +1,16 @@ +--- +title: Compliance Scan Guides +--- + + + + + +- [Install rancher-compliance](install-rancher-compliance.md) +- [Uninstall rancher-compliance](uninstall-rancher-compliance.md) +- [Run a Scan](run-a-scan.md) +- [Run a Scan Periodically on a Schedule](run-a-scan-periodically-on-a-schedule.md) +- [View Reports](view-reports.md) +- [Enable Alerting for rancher-compliance](enable-alerting-for-rancher-compliance.md) +- [Configure Alerts for Periodic Scan on a Schedule](configure-alerts-for-periodic-scan-on-a-schedule.md) +- [Create a Custom Benchmark Version to Run](create-a-custom-compliance-version-to-run.md) diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md similarity index 60% rename from versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md rename to docs/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md index 204f95c05bd..3f53f30dc9a 100644 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md @@ -3,7 +3,7 @@ title: Configure Alerts for Periodic Scan on a Schedule --- - + It is possible to run a ClusterScan on a schedule. @@ -12,27 +12,27 @@ A scheduled scan can also specify if you should receive alerts when the scan com Alerts are supported only for a scan that runs on a schedule. -The CIS Benchmark application supports two types of alerts: +The compliance application supports two types of alerts: - Alert on scan completion: This alert is sent out when the scan run finishes. The alert includes details including the ClusterScan's name and the ClusterScanProfile name. - Alert on scan failure: This alert is sent out if there are some test failures in the scan run or if the scan is in a `Fail` state. :::note Prerequisite -Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) +Before enabling alerts for `rancher-compliance`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) -While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) +While configuring the routes for `rancher-compliance` alerts, you can specify the matching using the key-value pair `job: rancher-compliance-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-compliance-scan-alerts) ::: To configure alerts for a scan that runs on a schedule, -1. Please enable alerts on the `rancher-cis-benchmark` application. For more information, see [this page](../../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md). +1. Please enable alerts on the `rancher-compliance` application. For more information, see [this page](../../../how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md). 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. +1. On the **Clusters** page, go to the cluster where you want to run a Compliance scan and click **Explore**. +1. Click **compliance > Scan**. 1. Click **Create**. -1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the Compliance Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. 1. Choose the option **Run scan on a schedule**. 1. Enter a valid [cron schedule expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) in the field **Schedule**. 1. Check the boxes next to the Alert types under **Alerting**. @@ -41,4 +41,4 @@ To configure alerts for a scan that runs on a schedule, **Result:** The scan runs and reschedules to run according to the cron schedule provided. Alerts are sent out when the scan finishes if routes and receiver are configured under `rancher-monitoring` application. -A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. \ No newline at end of file +A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. diff --git a/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run.md new file mode 100644 index 00000000000..a15fc96a7fd --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run.md @@ -0,0 +1,13 @@ +--- +title: Create a Custom Compliance Version for Running a Cluster Scan +--- + + + + + +There could be some Kubernetes cluster setups that require custom configurations of the Compliance tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream Compliance look for them. + +It is now possible to create a custom compliance version for running a cluster scan using the `rancher-compliance` application. + +For details, see [this page.](../../../integrations-in-rancher/compliance-scans/custom-benchmark.md) \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md new file mode 100644 index 00000000000..d5328a0dd0c --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md @@ -0,0 +1,24 @@ +--- +title: Enable Alerting for Rancher Compliance +--- + + + + + +Alerts can be configured to be sent out for a scan that runs on a schedule. + +:::note Prerequisite: + +Before enabling alerts for `rancher-compliance`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) + +While configuring the routes for `rancher-compliance` alerts, you can specify the matching using the key-value pair `job: rancher-compliance-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-compliance-scan-alerts) + +::: + +While installing or upgrading the `rancher-compliance` Helm chart, set the following flag to `true` in the `values.yaml`: + +```yaml +alerts: + enabled: true +``` \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance.md similarity index 51% rename from docs/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md rename to docs/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance.md index c6987a97c64..c00eab90648 100644 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance.md @@ -1,15 +1,15 @@ --- -title: Install Rancher CIS Benchmark +title: Install Rancher Compliance --- - + 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to install CIS Benchmark and click **Explore**. +1. On the **Clusters** page, go to the cluster where you want to install Compliance and click **Explore**. 1. In the left navigation bar, click **Apps > Charts**. -1. Click **CIS Benchmark** +1. Click **Compliance** 1. Click **Install**. -**Result:** The CIS scan application is deployed on the Kubernetes cluster. +**Result:** The compliance scan application is deployed on the Kubernetes cluster. diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule.md similarity index 75% rename from versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md rename to docs/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule.md index 076fbdf409b..ed0d251592e 100644 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule.md @@ -3,15 +3,15 @@ title: Run a Scan Periodically on a Schedule --- - + To run a ClusterScan on a schedule, 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. -1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. On the **Clusters** page, go to the cluster where you want to run a Compliance scan and click **Explore**. +1. Click **Compliance > Scan**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the Compliance Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. 1. Choose the option **Run scan on a schedule**. 1. Enter a valid cron schedule expression in the field **Schedule**. 1. Choose a **Retention** count, which indicates the number of reports maintained for this recurring scan. By default this count is 3. When this retention limit is reached, older reports will get purged. @@ -21,4 +21,4 @@ To run a ClusterScan on a schedule, A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. -You can also see the previous reports by choosing the report from the **Reports** dropdown on the scan detail page. \ No newline at end of file +You can also see the previous reports by choosing the report from the **Reports** dropdown on the scan detail page. diff --git a/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan.md new file mode 100644 index 00000000000..5b91cdb7e52 --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan.md @@ -0,0 +1,26 @@ +--- +title: Run a Scan +--- + + + + + +When a ClusterScan custom resource is created, it launches a new compliance scan on the cluster for the chosen ClusterScanProfile. + +:::note + +There is currently a limitation of running only one compliance scan at a time for a cluster. If you create multiple ClusterScan custom resources, they will be run one after the other by the operator, and until one scan finishes, the rest of the ClusterScan custom resources will be in the "Pending" state. + +::: + +To run a scan, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a compliance scan and click **Explore**. +1. Click **Compliance > Scan**. +1. Click **Create**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the Compliance Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Click **Create**. + +**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. diff --git a/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance.md new file mode 100644 index 00000000000..313acf79555 --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance.md @@ -0,0 +1,13 @@ +--- +title: Uninstall Rancher Compliance +--- + + + + + +1. From the **Cluster Dashboard,** go to the left navigation bar and click **Apps > Installed Apps**. +1. Go to the `compliance-operator-system` namespace and check the boxes next to `rancher-compliance-crd` and `rancher-compliance`. +1. Click **Delete** and confirm **Delete**. + +**Result:** The `rancher-compliance` application is uninstalled. \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports.md b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports.md new file mode 100644 index 00000000000..ad042390485 --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports.md @@ -0,0 +1,23 @@ +--- +title: View Reports +--- + + + + + +To view the generated Compliance scan reports, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a Compliance scan and click **Explore**. +1. Click **Compliance > Scan**. +1. The **Scans** page will show the generated reports. To see a detailed report, go to a scan report and click the name. + +One can download the report from the Scans list or from the scan detail page. + +To get the verbose version of the compliance scan results, run the following command on the cluster that was scanned. Note that the scan must be completed before this can be done. + +```console +export REPORT="scan-report-name" +kubectl get clusterscanreports.compliance.cattle.io $REPORT -o json |jq ".spec.reportJSON | fromjson" | jq -r ".actual_value_map_data" | base64 -d | gunzip | jq . +``` diff --git a/docs/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md b/docs/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md index f7266be3ff3..7a4384b4dad 100644 --- a/docs/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md +++ b/docs/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md @@ -185,6 +185,7 @@ For help troubleshooting certificates, see [this section.](../../getting-started If you want to record all transactions with the Rancher API, enable the [API Auditing](enable-api-audit-log.md) feature by adding the flags below into your install command. -e AUDIT_LEVEL=1 \ + -e AUDIT_LOG_ENABLED=true \ -e AUDIT_LOG_PATH=/var/log/auditlog/rancher-api-audit.log \ -e AUDIT_LOG_MAXAGE=20 \ -e AUDIT_LOG_MAXBACKUP=20 \ diff --git a/docs/how-to-guides/advanced-user-guides/configure-oidc-provider.md b/docs/how-to-guides/advanced-user-guides/configure-oidc-provider.md new file mode 100644 index 00000000000..f96a4fb92aa --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/configure-oidc-provider.md @@ -0,0 +1,138 @@ +--- +title: Configure Rancher as an OIDC provider +--- + + + + + +Rancher can function as a standard OpenID Connect (OIDC) provider, allowing external applications to use Rancher for authentication. +This can be used for enabling single sign-on (SSO) across Rancher Prime components. For example, see the [documentation](https://documentation.suse.com/cloudnative/suse-observability/next/en/setup/security/authentication/oidc.html) for configuring the OIDC provider for SUSE Observability. + +The OIDC provider can be enabled with the `oidc-provider` feature flag. When this flag is on the following endpoints are available: + +- `https://{rancher-url}/oidc/authorize`: This endpoint initiates the authentication flow. If a user is already logged into Rancher, it returns an authorization code. Otherwise, it redirects the user to the Rancher login page. Authorization codes and related request information are securely stored in session secrets. Codes are single-use and expire after 10 minutes. + +- `https://{rancher-url}/oidc/token`: This endpoint exchanges an authorization code for an `id_token`, `access_token`, and `refresh_token`. + +- `https://{rancher-url}/oidc/.well-known/openid-configuration`: This endpoint returns a JSON document containing the OIDC provider's configuration, including endpoint URLs, supported scopes, claims, and other relevant details. + +- `https://{rancher-url}/oidc/userinfo`: This endpoint provides information about the authenticated user. + +The OIDC provider supports the OIDC Authentication Code Flow with PKCE. + +## Configure OIDCClient + +An `OIDCClient` represents an external application that will be authenticating against Rancher. + +### Programmatically + +Create an `OIDCClient`: + +```yaml +apiVersion: management.cattle.io/v3 +kind: OIDCClient +metadata: + name: oidc-client-test +spec: + tokenExpirationSeconds: 600 # expiration of the id_token and access_token + refreshTokenExpirationSeconds: 3600 # expiration of the refresh_token + redirectURIs: + - "https://myredirecturl.com" # replace with your redirect url +``` +Rancher automatically generates a client ID and client secret for each `OIDCClient`. +Once the resource is created, Rancher populates the status field with the client id: + +```yaml +apiVersion: management.cattle.io/v3 +kind: OIDCClient +metadata: + name: oidc-client-test +spec: + tokenExpirationSeconds: 600 # expiration of the id_token and access_token + refreshTokenExpirationSeconds: 3600 # expiration of the refresh_token + redirectURIs: + - "https://myredirecturl.com" # replace with your redirect url +status: + clientID: client-xxx + clientSecrets: + client-secret-1: + createdAt: "xxx" + lastFiveCharacters: xxx +``` + +Rancher automatically generates a Kubernetes `Secret` in the `cattle-oidc-client-secrets` namespace for each `OIDCClient` resource. The Secret's name matches the `OIDCClient` client ID. +Initially, the `Secret` contains a single client secret. + +To retrieve the client secret: + +``` +kubectl get secret client-xxx -n cattle-oidc-client-secrets -o jsonpath="{.data.client-secret-1}" | base64 -d +``` + +Output: + +``` +secret-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +You can now use this client ID and client secret in your OIDC client application. + +#### Managing Client Secrets + +You can manage multiple client secrets per `OIDCClient`. Use annotations on the `OIDCClient` resource to perform secret operations: + +- Creation: Adding the `cattle.io/oidc-client-secret-create: true` annotation triggers the creation of a new client secret. +- Removal: Adding the `cattle.io/oidc-client-secret-remove:client-secret-1` annotation removes the specified client secrets. +- Regeneration: Adding the `cattle.io/oidc-client-secret-regenerate:client-secret-1` annotation regenerates the specified client secrets. + +### Rancher UI + +Create an OIDCClient: + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **OIDC Apps**. +1. Click **Add Application**. Fill out the **Create OIDC App** form. +1. Click **Add Application**. + +#### Managing Client Secrets + +In the OIDC App page: + +- Creation: Click **Add new secret**. +- Removal: Click **⋮ > Delete** +- Regeneration: Click **⋮ > Regenerate** + +## Signing key + +A default key pair for signing the `id_token`, `access_token`, and `refresh_token` tokens is created by Rancher in a `Secret` called `oidc-signing-key` in the `cattle-system` namespace. Only one key will be used for signing, but multiple public keys can be returned in the jwks endpoint in order to avoid disruption when doing a key rotation. + +### Rotation without disruption + +In order to create a new key pair for signing you need to manually create a new keypair and add it to the `oidc-signing-key` `Secret` + +Example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: oidc-signing-key +type: Opaque +data: + key2.pem: + key1.pub: + key2.pub: +``` + +Rancher will sign tokens using `key2.pem`, while the JWKS endpoint will serve both `key1.pub` and `key2.pub`. This ensures a smooth +key rotation from `key1` to `key2` without disrupting existing token verification. Note that only one private key (.pem) can be stored in the +secret at a time, and each key pair must share the same base name, differing only by their suffix: .pem for the private key and .pub for the public key. + +### Rotation with disruption + +Removing the `oidc-signing-key` `Secret` will cause Rancher to regenerate the signing key on the next restart. + +:::warning +This will invalidate all previously issued `id_token`, `access_token`, and `refresh_token` tokens making them unusable. +::: diff --git a/docs/how-to-guides/advanced-user-guides/enable-api-audit-log.md b/docs/how-to-guides/advanced-user-guides/enable-api-audit-log.md index 4f0a15c7984..36e9a7f5ad3 100644 --- a/docs/how-to-guides/advanced-user-guides/enable-api-audit-log.md +++ b/docs/how-to-guides/advanced-user-guides/enable-api-audit-log.md @@ -24,7 +24,8 @@ The usage below defines rules about what the audit log should record and what da | Parameter | Description | | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `AUDIT_LEVEL` | `0` - Disable audit log (default setting).
`1` - Log event metadata.
`2` - Log event metadata and request body.
`3` - Log event metadata, request body, and response body. Each log transaction for a request/response pair uses the same `auditID` value.

See [Audit Level Logging](#audit-log-levels) for a table that displays what each setting logs. | +| `AUDIT_LOG_ENABLED` | `false` - Disables the audit log (default setting).
`true` - Enables the audit log. | +| `AUDIT_LEVEL` | `0` - Log request and response metadata (default setting).
`1` - Log request and response headers.
`2` - Log request body.
`3` - Log response body. Each log level is cumulative and each subsequent level logs the previous level data. Each log transaction for a request/response pair uses the same `auditID` value.

See [Audit Level Logging](#audit-log-levels) for a table that displays what each setting logs. | | `AUDIT_LOG_PATH` | Log path for Rancher Server API. Default path is `/var/log/auditlog/rancher-api-audit.log`. You can mount the log directory to host.

Usage Example: `AUDIT_LOG_PATH=/my/custom/path/`
| | `AUDIT_LOG_MAXAGE` | Defined the maximum number of days to retain old audit log files. Default is 10 days. | | `AUDIT_LOG_MAXBACKUP` | Defines the maximum number of audit log files to retain. Default is 10. | @@ -36,12 +37,138 @@ The usage below defines rules about what the audit log should record and what da The following table displays what parts of API transactions are logged for each [`AUDIT_LEVEL`](#api-audit-log-options) setting. -| `AUDIT_LEVEL` Setting | Metadata | Request Body | Response Body | -| --------------------- | -------- | ------------ | ------------- | -| `0` | | | | -| `1` | ✓ | | | -| `2` | ✓ | ✓ | | -| `3` | ✓ | ✓ | ✓ | + +| `AUDIT_LEVEL` Setting | Metadata | Request Headers | Response Headers | Request Body | Response Body | +|--------------------------------|----------------|---------------------------|-----------------------------|----------------------|-----------------------| +| 0 | ✓ | +| 1 | ✓ | ✓ | ✓ +| 2 | ✓ | ✓ | ✓ | ✓ +| 3 | ✓ | ✓ | ✓ | ✓ | ✓ + +## Audit Log policies + +Audit log policies allow end users to configure redactions using `AuditPolicy` cluster-scoped CRs in addition to the [default redactions and filters](#default-redactions--filters). + +All configured audit log policies are additive. + +Redaction policies for headers utilize a regular expression (regex) engine to redact headers, while a JSONPath engine is used to redact request/response headers. + +The JSONPath engine does not support the script or filter expressions. For getting started with JSONPath expressions, a good resource to consult is [Stafan Goessner's article on JSONPath](https://goessner.net/articles/JsonPath/). + +The structure of an audit policy CR is as follows: + +```yaml +apiVersion: auditlog.cattle.io/v1 +kind: AuditPolicy +spec: + enabled : true # true/false + # list of API request filters + filters: + - action: allow # allow/deny + # would allow logs sent to "/foo/some/endpoint" but not "/foo" or "/foobar". + requestURI: "/foo/.*" + # additionalRedactions allows configuration of redactions on headers using `jsonpath` expressions + additionalRedactions: + # redacts headers based on regex expressions + - headers: + - "Cache.*" + # paths redacts information from request and response bodies based on json path expressions + paths: + - "$.gitCommit" + verbosity: + level : 0 # matches the levels in the above audit log table + # request allows fine-grained control over which request data + # gets included. This overrides the behaviour of the generic verbosity.level + request: + headers : true # true/false + body : true # true/false + # response allows fine-grained control over which response data + # gets included. This overrides the behaviour of the generic verbosity.level + response: + headers : true # true/false + body: true # true/false +``` + +### Examples + +The following example shows only logging requests containing `login` in the request path to the audit log path: + +```yaml +apiVersion: auditlog.cattle.io/v1 +kind: AuditPolicy +metadata: + name: filters +spec: + filters: + - action: deny + requestUri: ".*" + - action: allow + requestUri: ".*login.*" +``` + +The following example shows how to redact specific fields containing `gitCommint` in request/response bodies: + +```yaml +apiVersion: auditlog.cattle.io/v1 +kind: AuditPolicy +metadata: + name: redactions +spec: + additionalRedactions: + - paths: + - "$.gitCommit" +``` + +### Default redactions & filters + +The audit log controller comes with default built-in redactions for common sensitive information. + +#### Redacted headers + +Generic headers: +- `Cookie` +- `Set-Cookie` +- `X-Api-Set-Cookie-Header` +- `Authorization` +- `X-Api-Tunnel-Params` +- `X-Api-Tunnel-Token` +- `X-Api-Auth-Header` +- `X-Amz-Security-Token` + + +#### Redacted body fields + +Generic body fields: + +- `credentials` +- `applicationSecret` +- `oauthCredential` +- `serviceAccountCredential` +- `spKey` +- `spCert` +- `certificate` +- `privateKey` +- `secretsEncryptionConfig` +- `manifestUrl` +- `insecureWindowsNodeCommand` +- `insecureNodeCommand` +- `insecureCommand` +- `command` +- `nodeCommand` +- `windowsNodeCommand` +- `clientRandom` + +Generic body regex redactor: +- `".*([pP]assword|[Kk]ube[Cc]onfig|[Tt]oken).*"` + +#### Cluster Driver + +By default, any API request with fields tied to cluster drivers will have any non `public*` or `optional*` fields redacted by the audit log controller. + +#### Redacted URIs + +Any endpoint containing `secrets` or `configmaps` will have relevant fields stripped from both the request and response bodies. Additionally, any endpoint containing `/v3/imports/*` will have its URI redacted. + ## Viewing API Audit Logs @@ -71,492 +198,488 @@ After you enable auditing, each API request or response is logged by Rancher in ### Metadata Level -If you set your `AUDIT_LEVEL` to `1`, Rancher logs the metadata header for every API request, but not the body. The header provides basic information about the API transaction, such as the transaction's ID, who initiated the transaction, the time it occurred, etc. +If you set your `AUDIT_LEVEL` to `0`, Rancher logs the metadata header for every API request, but neither the body nor the request and response headers. The metadata provides basic information about the API transaction, such as the transaction ID, the initiator of the transaction, the time it occurred, etc. ```json { - "auditID": "30022177-9e2e-43d1-b0d0-06ef9d3db183", - "requestURI": "/v3/schemas", - "sourceIPs": ["::1"], - "user": { - "name": "user-f4tt2", - "group": ["system:authenticated"] - }, - "verb": "GET", - "stage": "RequestReceived", - "stageTimestamp": "2018-07-20 10:22:43 +0800" + "auditID": "40bd4e40-875b-4020-933e-4c4f4c4db366", + "requestURI": "/v3/schemas", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] + } + }, + "method": "GET", + "remoteAddr": "127.0.0.1:58652", + "responseCode": 200, + "requestTimestamp": "2025-06-30T11:13:25-04:00", + "responseTimestamp": "2025-06-30T11:13:25-04:00" } ``` -### Metadata and Request Body Level +### Metadata and headers level -If you set your `AUDIT_LEVEL` to `2`, Rancher logs the metadata header and body for every API request. - -The code sample below depicts an API request, with both its metadata header and body. +If you set your `AUDIT_LEVEL` to `1`, Rancher logs the metadata and the request and response headers for every API request. ```json { - "auditID": "ef1d249e-bfac-4fd0-a61f-cbdcad53b9bb", - "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "sourceIPs": ["::1"], - "user": { - "name": "user-f4tt2", - "group": ["system:authenticated"] - }, - "verb": "PUT", - "stage": "RequestReceived", - "stageTimestamp": "2018-07-20 10:28:08 +0800", - "requestBody": { - "hostIPC": false, - "hostNetwork": false, - "hostPID": false, - "paused": false, - "annotations": {}, - "baseType": "workload", - "containers": [ - { - "allowPrivilegeEscalation": false, - "image": "nginx", - "imagePullPolicy": "Always", - "initContainer": false, - "name": "nginx", - "ports": [ - { - "containerPort": 80, - "dnsName": "nginx-nodeport", - "kind": "NodePort", - "name": "80tcp01", - "protocol": "TCP", - "sourcePort": 0, - "type": "/v3/project/schemas/containerPort" - } + "auditID": "f8c83dc6-a080-4e2e-ab43-552bddf01716", + "requestURI": "/v1/apps.deployments?page=1&pagesize=100&sort=metadata.name&filter=metadata.namespace!=p-npsl5&filter=metadata.namespace!=p-nzp6c&filter=metadata.namespace!=cattle-fleet-clusters-system&filter=metadata.namespace!=cattle-fleet-system&filter=metadata.namespace!=cattle-global-data&filter=metadata.namespace!=cattle-impersonation-system&filter=metadata.namespace!=cattle-provisioning-capi-system&filter=metadata.namespace!=cattle-system&filter=metadata.namespace!=cattle-ui-plugin-system&filter=metadata.namespace!=cluster-fleet-local-local-1a3d67d0a899&filter=metadata.namespace!=fleet-default&filter=metadata.namespace!=fleet-local&filter=metadata.namespace!=kube-node-lease&filter=metadata.namespace!=kube-public&filter=metadata.namespace!=kube-system&exclude=metadata.managedFields", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" ], - "privileged": false, - "readOnly": false, - "resources": { - "type": "/v3/project/schemas/resourceRequirements", - "requests": {}, - "limits": {} - }, - "restartCount": 0, - "runAsNonRoot": false, - "stdin": true, - "stdinOnce": false, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "tty": true, - "type": "/v3/project/schemas/container", - "environmentFrom": [], - "capAdd": [], - "capDrop": [], - "livenessProbe": null, - "volumeMounts": [] - } - ], - "created": "2018-07-18T07:34:16Z", - "createdTS": 1531899256000, - "creatorId": null, - "deploymentConfig": { - "maxSurge": 1, - "maxUnavailable": 0, - "minReadySeconds": 0, - "progressDeadlineSeconds": 600, - "revisionHistoryLimit": 10, - "strategy": "RollingUpdate" - }, - "deploymentStatus": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2018-07-18T07:34:38Z", - "lastTransitionTimeTS": 1531899278000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - }, - { - "lastTransitionTime": "2018-07-18T07:34:16Z", - "lastTransitionTimeTS": 1531899256000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", - "reason": "NewReplicaSetAvailable", - "status": "True", - "type": "Progressing" + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "type": "/v3/project/schemas/deploymentStatus", - "unavailableReplicas": 0, - "updatedReplicas": 1 }, - "dnsPolicy": "ClusterFirst", - "id": "deployment:default:nginx", - "labels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "name": "nginx", - "namespaceId": "default", - "projectId": "c-bcz5t:p-fdr4s", - "publicEndpoints": [ - { - "addresses": ["10.64.3.58"], - "allNodes": true, - "ingressId": null, - "nodeId": null, - "podId": null, - "port": 30917, - "protocol": "TCP", - "serviceId": "default:nginx-nodeport", - "type": "publicEndpoint" - } - ], - "restartPolicy": "Always", - "scale": 1, - "schedulerName": "default-scheduler", - "selector": { - "matchLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "type": "/v3/project/schemas/labelSelector" - }, - "state": "active", - "terminationGracePeriodSeconds": 30, - "transitioning": "no", - "transitioningMessage": "", - "type": "deployment", - "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", - "workloadAnnotations": { - "deployment.kubernetes.io/revision": "1", - "field.cattle.io/creatorId": "user-f4tt2" - }, - "workloadLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "scheduling": { - "node": {} - }, - "description": "my description", - "volumes": [] - } -} -``` - -### Metadata, Request Body, and Response Body Level - -If you set your `AUDIT_LEVEL` to `3`, Rancher logs: - -- The metadata header and body for every API request. -- The metadata header and body for every API response. - -#### Request - -The code sample below depicts an API request, with both its metadata header and body. - -```json -{ - "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", - "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "sourceIPs": ["::1"], - "user": { - "name": "user-f4tt2", - "group": ["system:authenticated"] - }, - "verb": "PUT", - "stage": "RequestReceived", - "stageTimestamp": "2018-07-20 10:33:06 +0800", - "requestBody": { - "hostIPC": false, - "hostNetwork": false, - "hostPID": false, - "paused": false, - "annotations": {}, - "baseType": "workload", - "containers": [ - { - "allowPrivilegeEscalation": false, - "image": "nginx", - "imagePullPolicy": "Always", - "initContainer": false, - "name": "nginx", - "ports": [ - { - "containerPort": 80, - "dnsName": "nginx-nodeport", - "kind": "NodePort", - "name": "80tcp01", - "protocol": "TCP", - "sourcePort": 0, - "type": "/v3/project/schemas/containerPort" - } + "method": "GET", + "remoteAddr": "127.0.0.1:58833", + "responseCode": 200, + "requestTimestamp": "2025-06-30T11:17:04-04:00", + "responseTimestamp": "2025-06-30T11:17:04-04:00", + "requestHeader": { + "Accept": [ + "application/json" ], - "privileged": false, - "readOnly": false, - "resources": { - "type": "/v3/project/schemas/resourceRequirements", - "requests": {}, - "limits": {} - }, - "restartCount": 0, - "runAsNonRoot": false, - "stdin": true, - "stdinOnce": false, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "tty": true, - "type": "/v3/project/schemas/container", - "environmentFrom": [], - "capAdd": [], - "capDrop": [], - "livenessProbe": null, - "volumeMounts": [] - } - ], - "created": "2018-07-18T07:34:16Z", - "createdTS": 1531899256000, - "creatorId": null, - "deploymentConfig": { - "maxSurge": 1, - "maxUnavailable": 0, - "minReadySeconds": 0, - "progressDeadlineSeconds": 600, - "revisionHistoryLimit": 10, - "strategy": "RollingUpdate" - }, - "deploymentStatus": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2018-07-18T07:34:38Z", - "lastTransitionTimeTS": 1531899278000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - }, - { - "lastTransitionTime": "2018-07-18T07:34:16Z", - "lastTransitionTimeTS": 1531899256000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", - "reason": "NewReplicaSetAvailable", - "status": "True", - "type": "Progressing" - } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "type": "/v3/project/schemas/deploymentStatus", - "unavailableReplicas": 0, - "updatedReplicas": 1 - }, - "dnsPolicy": "ClusterFirst", - "id": "deployment:default:nginx", - "labels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "name": "nginx", - "namespaceId": "default", - "projectId": "c-bcz5t:p-fdr4s", - "publicEndpoints": [ - { - "addresses": ["10.64.3.58"], - "allNodes": true, - "ingressId": null, - "nodeId": null, - "podId": null, - "port": 30917, - "protocol": "TCP", - "serviceId": "default:nginx-nodeport", - "type": "publicEndpoint" - } - ], - "restartPolicy": "Always", - "scale": 1, - "schedulerName": "default-scheduler", - "selector": { - "matchLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "type": "/v3/project/schemas/labelSelector" - }, - "state": "active", - "terminationGracePeriodSeconds": 30, - "transitioning": "no", - "transitioningMessage": "", - "type": "deployment", - "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", - "workloadAnnotations": { - "deployment.kubernetes.io/revision": "1", - "field.cattle.io/creatorId": "user-f4tt2" - }, - "workloadLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "scheduling": { - "node": {} - }, - "description": "my decript", - "volumes": [] - } -} -``` - -#### Response - -The code sample below depicts an API response, with both its metadata header and body. - -```json -{ - "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", - "responseStatus": "200", - "stage": "ResponseComplete", - "stageTimestamp": "2018-07-20 10:33:06 +0800", - "responseBody": { - "actionLinks": { - "pause": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=pause", - "resume": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=resume", - "rollback": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=rollback" - }, - "annotations": {}, - "baseType": "workload", - "containers": [ - { - "allowPrivilegeEscalation": false, - "image": "nginx", - "imagePullPolicy": "Always", - "initContainer": false, - "name": "nginx", - "ports": [ - { - "containerPort": 80, - "dnsName": "nginx-nodeport", - "kind": "NodePort", - "name": "80tcp01", - "protocol": "TCP", - "sourcePort": 0, - "type": "/v3/project/schemas/containerPort" - } + "Accept-Encoding": [ + "gzip, deflate, br, zstd" ], - "privileged": false, - "readOnly": false, - "resources": { - "type": "/v3/project/schemas/resourceRequirements" - }, - "restartCount": 0, - "runAsNonRoot": false, - "stdin": true, - "stdinOnce": false, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "tty": true, - "type": "/v3/project/schemas/container" - } - ], - "created": "2018-07-18T07:34:16Z", - "createdTS": 1531899256000, - "creatorId": null, - "deploymentConfig": { - "maxSurge": 1, - "maxUnavailable": 0, - "minReadySeconds": 0, - "progressDeadlineSeconds": 600, - "revisionHistoryLimit": 10, - "strategy": "RollingUpdate" + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "Connection": [ + "keep-alive" + ], + "Cookie": [ + "[redacted]" + ], + "Referer": [ + "https://localhost:8443/dashboard/c/local/explorer/apps.deployment" + ], + "Sec-Fetch-Dest": [ + "empty" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "Sec-Fetch-Site": [ + "same-origin" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0" + ], + "X-Api-Csrf": [ + "fccc690cab7b0c169b3fc6527edadef3" + ] }, - "deploymentStatus": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2018-07-18T07:34:38Z", - "lastTransitionTimeTS": 1531899278000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - }, - { - "lastTransitionTime": "2018-07-18T07:34:16Z", - "lastTransitionTimeTS": 1531899256000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", - "reason": "NewReplicaSetAvailable", - "status": "True", - "type": "Progressing" - } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "type": "/v3/project/schemas/deploymentStatus", - "unavailableReplicas": 0, - "updatedReplicas": 1 - }, - "dnsPolicy": "ClusterFirst", - "hostIPC": false, - "hostNetwork": false, - "hostPID": false, - "id": "deployment:default:nginx", - "labels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "links": { - "remove": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "revisions": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/revisions", - "self": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "update": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "yaml": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/yaml" - }, - "name": "nginx", - "namespaceId": "default", - "paused": false, - "projectId": "c-bcz5t:p-fdr4s", - "publicEndpoints": [ - { - "addresses": ["10.64.3.58"], - "allNodes": true, - "ingressId": null, - "nodeId": null, - "podId": null, - "port": 30917, - "protocol": "TCP", - "serviceId": "default:nginx-nodeport" - } - ], - "restartPolicy": "Always", - "scale": 1, - "schedulerName": "default-scheduler", - "selector": { - "matchLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "type": "/v3/project/schemas/labelSelector" - }, - "state": "active", - "terminationGracePeriodSeconds": 30, - "transitioning": "no", - "transitioningMessage": "", - "type": "deployment", - "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", - "workloadAnnotations": { - "deployment.kubernetes.io/revision": "1", - "field.cattle.io/creatorId": "user-f4tt2" - }, - "workloadLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + "responseHeader": { + "Cache-Control": [ + "no-cache, no-store, must-revalidate" + ], + "Content-Encoding": [ + "gzip" + ], + "Content-Type": [ + "application/json" + ], + "Expires": [ + "Wed 24 Feb 1982 18:42:00 GMT" + ], + "X-Api-Cattle-Auth": [ + "true" + ], + "X-Api-Schemas": [ + "https://localhost:8443/v1/schemas" + ], + "X-Content-Type-Options": [ + "nosniff" + ] } - } } ``` + +### Metadata, headers and Request Body Level + +If you set your `AUDIT_LEVEL` to `2`, Rancher logs the metadata, the request and response headers and the request body for every API request. + +The code sample below depicts an API request, with both its metadata, headers and request body. + +```json +{ + "auditID": "d1088a09-2a13-4450-970e-0d44bd2c49ee", + "requestURI": "/v3/projects", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] + } + }, + "method": "POST", + "remoteAddr": "127.0.0.1:49966", + "responseCode": 201, + "requestTimestamp": "2025-06-30T12:32:13-04:00", + "responseTimestamp": "2025-06-30T12:32:13-04:00", + "requestHeader": { + "Accept": [ + "application/json" + ], + "Accept-Encoding": [ + "gzip, deflate, br, zstd" + ], + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "214" + ], + "Content-Type": [ + "application/json" + ], + "Cookie": [ + "[redacted]" + ], + "Impersonate-Extra-Principalid": [ + "local://user-6j5s6" + ], + "Impersonate-Extra-Requesthost": [ + "localhost:8443" + ], + "Impersonate-Extra-Requesttokenid": [ + "token-zs42h" + ], + "Impersonate-Extra-Username": [ + "admin" + ], + "Impersonate-Group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "Impersonate-User": [ + "user-6j5s6" + ], + "Origin": [ + "https://localhost:8443" + ], + "Priority": [ + "u=0" + ], + "Referer": [ + "https://localhost:8443/dashboard/c/local/explorer/management.cattle.io.project/create" + ], + "Sec-Fetch-Dest": [ + "empty" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "Sec-Fetch-Site": [ + "same-origin" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0" + ], + "X-Api-Csrf": [ + "fccc690cab7b0c169b3fc6527edadef3" + ] + }, + "responseHeader": { + "Cache-Control": [ + "no-cache, no-store, must-revalidate" + ], + "Content-Encoding": [ + "gzip" + ], + "Content-Type": [ + "application/json" + ], + "Expires": [ + "Wed 24 Feb 1982 18:42:00 GMT" + ], + "X-Api-Cattle-Auth": [ + "true" + ], + "X-Api-Schemas": [ + "https://localhost:8443/v3/project/schemas" + ], + "X-Content-Type-Options": [ + "nosniff" + ] + }, + "requestBody": { + "annotations": {}, + "clusterId": "local", + "containerDefaultResourceLimit": {}, + "creatorId": "local://user-6j5s6", + "labels": {}, + "name": "example-project", + "namespaceDefaultResourceQuota": {}, + "resourceQuota": {}, + "type": "project" + } +} +``` + +### Metadata, Headers, Request Body and Response Body Level + +If you set your `AUDIT_LEVEL` to `3`, Rancher logs the metadata, request and response headers and request body and response. + +The code sample below depicts an example of an API request with that information logged. + +```json +{ + "auditID": "a9549a5b-4351-4bd5-adcd-12f7ec667a6b", + "requestURI": "/v3/projects", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] + } + }, + "method": "POST", + "remoteAddr": "127.0.0.1:50454", + "responseCode": 201, + "requestTimestamp": "2025-06-30T12:42:24-04:00", + "responseTimestamp": "2025-06-30T12:42:24-04:00", + "requestHeader": { + "Accept": [ + "application/json" + ], + "Accept-Encoding": [ + "gzip, deflate, br, zstd" + ], + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "214" + ], + "Content-Type": [ + "application/json" + ], + "Cookie": [ + "[redacted]" + ], + "Impersonate-Extra-Principalid": [ + "local://user-6j5s6" + ], + "Impersonate-Extra-Requesthost": [ + "localhost:8443" + ], + "Impersonate-Extra-Requesttokenid": [ + "token-zs42h" + ], + "Impersonate-Extra-Username": [ + "admin" + ], + "Impersonate-Group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "Impersonate-User": [ + "user-6j5s6" + ], + "Origin": [ + "https://localhost:8443" + ], + "Priority": [ + "u=0" + ], + "Referer": [ + "https://localhost:8443/dashboard/c/local/explorer/management.cattle.io.project/create" + ], + "Sec-Fetch-Dest": [ + "empty" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "Sec-Fetch-Site": [ + "same-origin" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0" + ], + "X-Api-Csrf": [ + "fccc690cab7b0c169b3fc6527edadef3" + ] + }, + "responseHeader": { + "Cache-Control": [ + "no-cache, no-store, must-revalidate" + ], + "Content-Encoding": [ + "gzip" + ], + "Content-Type": [ + "application/json" + ], + "Expires": [ + "Wed 24 Feb 1982 18:42:00 GMT" + ], + "X-Api-Cattle-Auth": [ + "true" + ], + "X-Api-Schemas": [ + "https://localhost:8443/v3/project/schemas" + ], + "X-Content-Type-Options": [ + "nosniff" + ] + }, + "requestBody": { + "annotations": {}, + "clusterId": "local", + "containerDefaultResourceLimit": {}, + "creatorId": "local://user-6j5s6", + "labels": {}, + "name": "example-project", + "namespaceDefaultResourceQuota": {}, + "resourceQuota": {}, + "type": "project" + }, + "responseBody": { + "actions": { + "exportYaml": "https://localhost:8443/v3/projects/local:p-qt6tq?action=exportYaml" + }, + "annotations": { + "authz.management.cattle.io/creator-role-bindings": "{\"required\":[\"project-owner\"]}" + }, + "backingNamespace": "local-p-qt6tq", + "baseType": "project", + "clusterId": "local", + "containerDefaultResourceLimit": { + "type": "/v3/schemas/containerResourceLimit" + }, + "created": "2025-06-30T16:42:24Z", + "createdTS": 1751301744000, + "creatorId": "user-6j5s6", + "id": "local:p-qt6tq", + "labels": { + "cattle.io/creator": "norman" + }, + "links": { + "basicAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/basicauths", + "certificates": "https://localhost:8443/v3/projects/local:p-qt6tq/certificates", + "configMaps": "https://localhost:8443/v3/projects/local:p-qt6tq/configmaps", + "cronJobs": "https://localhost:8443/v3/projects/local:p-qt6tq/cronjobs", + "daemonSets": "https://localhost:8443/v3/projects/local:p-qt6tq/daemonsets", + "deployments": "https://localhost:8443/v3/projects/local:p-qt6tq/deployments", + "dnsRecords": "https://localhost:8443/v3/projects/local:p-qt6tq/dnsrecords", + "dockerCredentials": "https://localhost:8443/v3/projects/local:p-qt6tq/dockercredentials", + "horizontalPodAutoscalers": "https://localhost:8443/v3/projects/local:p-qt6tq/horizontalpodautoscalers", + "ingresses": "https://localhost:8443/v3/projects/local:p-qt6tq/ingresses", + "jobs": "https://localhost:8443/v3/projects/local:p-qt6tq/jobs", + "namespacedBasicAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedbasicauths", + "namespacedCertificates": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedcertificates", + "namespacedDockerCredentials": "https://localhost:8443/v3/projects/local:p-qt6tq/namespaceddockercredentials", + "namespacedSecrets": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedsecrets", + "namespacedServiceAccountTokens": "[redacted]", + "namespacedSshAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedsshauths", + "persistentVolumeClaims": "https://localhost:8443/v3/projects/local:p-qt6tq/persistentvolumeclaims", + "pods": "https://localhost:8443/v3/projects/local:p-qt6tq/pods", + "projectNetworkPolicies": "https://localhost:8443/v3/projects/local:p-qt6tq/projectnetworkpolicies", + "projectRoleTemplateBindings": "https://localhost:8443/v3/projects/local:p-qt6tq/projectroletemplatebindings", + "remove": "https://localhost:8443/v3/projects/local:p-qt6tq", + "replicaSets": "https://localhost:8443/v3/projects/local:p-qt6tq/replicasets", + "replicationControllers": "https://localhost:8443/v3/projects/local:p-qt6tq/replicationcontrollers", + "secrets": "https://localhost:8443/v3/projects/local:p-qt6tq/secrets", + "self": "https://localhost:8443/v3/projects/local:p-qt6tq", + "serviceAccountTokens": "[redacted]", + "services": "https://localhost:8443/v3/projects/local:p-qt6tq/services", + "sshAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/sshauths", + "statefulSets": "https://localhost:8443/v3/projects/local:p-qt6tq/statefulsets", + "subscribe": "https://localhost:8443/v3/projects/local:p-qt6tq/subscribe", + "update": "https://localhost:8443/v3/projects/local:p-qt6tq", + "workloads": "https://localhost:8443/v3/projects/local:p-qt6tq/workloads" + }, + "name": "example-project", + "namespaceDefaultResourceQuota": { + "limit": { + "type": "/v3/schemas/resourceQuotaLimit" + }, + "type": "/v3/schemas/namespaceResourceQuota" + }, + "namespaceId": null, + "resourceQuota": { + "limit": { + "type": "/v3/schemas/resourceQuotaLimit" + }, + "type": "/v3/schemas/projectResourceQuota", + "usedLimit": { + "type": "/v3/schemas/resourceQuotaLimit" + } + }, + "state": "active", + "transitioning": "no", + "transitioningMessage": "", + "type": "project", + "uuid": "b582603b-7826-4302-8393-792df2611265" + } +} +``` \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md b/docs/how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md deleted file mode 100644 index f3faaf47ceb..00000000000 --- a/docs/how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: UI Server-Side Pagination ---- - - - - - -:::caution -UI server-side pagination is not intended for use in production at this time. This feature is considered highly experimental. SUSE customers should consult SUSE Support before activating this feature. -::: - - -UI server-side pagination caching provides an optional SQLite-backed cache of Kubernetes objects to improve performance. This unlocks sorting, filtering and pagination features used by the UI to restrict the amount of resources it fetches and stores in browser memory. These features are primarily used to improve list performance for resources with high counts. - -This feature creates file system based caches in the `rancher` pods of the upstream cluster, and in the `cattle-cluster-agent` pods of the downstream clusters. In most environments, disk usage and I/O should not be significant. However, you should monitor activity after you enable caching. - -SQLite-backed caching persists copies of any cached Kubernetes objects to disk. See [Encrypting SQLite-backed Caching](#encrypting-sqlite-backed-caches) if this is a security concern. - -## Enabling UI Server-Side Pagination - -1. In the upper left corner, click **☰ > Global Settings > Feature Flags**. -1. Find **`ui-sql-cache`** and select **⋮ > Activate > Activate**. -1. Wait for Rancher to restart. This also restarts agents on all downstream clusters. -1. In the upper left corner, click **☰ > Global Settings > Performance**. -1. Go to **Server-side Pagination** and check the **Enable Server-side Pagination** option. -1. Click **Apply**. -1. Reload the page with the browser button (or the equivalent keyboard combination, typically `CTRL + R` on Windows and Linux, and `⌘ + R` on macOS). - - -## Encrypting SQLite-backed Caches - -UI server-side pagination persists copies of any cached Kubernetes objects to disk. If you're concerned about the safety of this data, you can encrypt all objects before they are persisted to disk, by setting the environment variable `CATTLE_ENCRYPT_CACHE_ALL` to `true` in `rancher` pods in the upstream cluster and `cattle-cluster-agent` pods in the downstream clusters. - -Secrets and security Tokens are always encrypted regardless of the above setting. - -## Known Limitations of UI Server-Side Pagination - -This initial release improves the performance of Pods, Secrets, Nodes and ConfigMaps in the Cluster Explorer pages, and most resources in the Explorer's **More Resources** section. - -Pages can't be automatically refreshed. You can manually refresh table contents by clicking the **Refresh** button. diff --git a/docs/how-to-guides/advanced-user-guides/ui-server-side-pagination.md b/docs/how-to-guides/advanced-user-guides/ui-server-side-pagination.md new file mode 100644 index 00000000000..d9bdef91c50 --- /dev/null +++ b/docs/how-to-guides/advanced-user-guides/ui-server-side-pagination.md @@ -0,0 +1,80 @@ +--- +title: UI Server-Side Pagination +--- + + + + + +Server-Side Pagination (SSP) is a Rancher feature to provide significant performance improvements across the UI for resources with high counts, restricting the amount of resources browser fetches and stores in memory. + +Note that SSP is optional, **enabled by default**, and it can be disabled via the feature flag `ui-sql-cache`. + +## Disk Space + +:::important +It is crucial that you review the available disk space on your nodes and plan accordingly before upgrading to Rancher v2.12.0 and later to avoid potential disk pressure and pod eviction issues. +::: + +The SSP relies on a caching mechanism that introduces a new requirement for ephemeral disk space on your cluster nodes. This cache, an internal SQLite database, is stored within the container's file system. This affects the nodes running the **Rancher server pods** (`rancher` in the `cattle-system` namespace on the local cluster) and the nodes running the **Rancher agent pods** (`cattle-cluster-agent` in the `cattle-system` namespace on all downstream clusters). + +The amount of disk space required is dynamic and depends on the quantity and size of Kubernetes resources visualized in the UI. As a guideline, the cache may consume approximately **twice the size of the raw Kubernetes objects** it stores. + +For example, internal tests showed that caching 5000 ConfigMaps, totaling 50 MB, consumed 81 MB of disk space. For a conservative, high-level estimate, you can plan for the available disk space on each relevant node to be at least **twice the size of your etcd snapshot**. For most production environments, ensuring a few extra gigabytes of storage are available on the relevant nodes is a safe starting point. + +Please note this space counts against [ephemeral storage](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#setting-requests-and-limits-for-local-ephemeral-storage) requests and limits you might have set for your Rancher container via the `resource` value in the Helm chart. Make sure those settings provide for abundant available space. + +If you see the error `database or disk is full (13)` in the pod logs, this is a symptom that more space needs to be allocated. + +SQLite-backed caching persists copies of any cached Kubernetes objects to disk. See [Encrypting SQLite-backed Caching](#encrypting-sqlite-backed-caches) if this is a security concern. + +## Enabling Server-Side Pagination + +1. In the upper left corner, click **☰ > Global Settings > Feature Flags**. +1. Find **`ui-sql-cache`** and select **⋮ > Activate > Activate**. +1. Wait for Rancher to restart. This also restarts agents on all downstream clusters. +1. Reload the page with the browser button (or the equivalent keyboard combination, typically `CTRL + R` on Windows and Linux, and `⌘ + R` on macOS). + +## Disabling Server-Side Pagination + +1. In the upper left corner, click **☰ > Global Settings > Feature Flags**. +1. Find **`ui-sql-cache`** and select **⋮ > Deactivate > Deactivate**. +1. Wait for Rancher to restart. This also restarts agents on all downstream clusters. +1. Reload the page with the browser button (or the equivalent keyboard combination, typically `CTRL + R` on Windows and Linux, and `⌘ + R` on macOS). + +## Encrypting SQLite-backed Caches + +UI server-side pagination persists copies of any cached Kubernetes objects to disk. If you're concerned about the safety of this data, you can encrypt all objects before they are persisted to disk, by setting the environment variable `CATTLE_ENCRYPT_CACHE_ALL` to `true` in `rancher` pods in the upstream cluster and `cattle-cluster-agent` pods in the downstream clusters. + +Secrets and security Tokens are always encrypted regardless of the above setting. + +## Known Limitations of UI Server-Side Pagination + +This release improves the performance of most pages used to view, create or edit resources within the `local` or downstream clusters i.e. the Cluster Explorer view. However, RBAC related resources and areas outside the Cluster Explorer are not yet covered by this feature. + +Additionally, the following limitations are present when the feature is enabled. These mainly revolve around different sort or filter behaviors in affected lists: + +- Resources in lists are automatically updated, however, not instantaneously. +- All lists that utilize Server-Side Pagination: + - `State` column sort and filter features work on the resources `metadata.state.name` field instead of one deduced locally by the UI. + - Updates are shown every 5 seconds, rather than instantly. +- Cluster Explorer: + - `Cluster` group --> `Nodes` page + - The following columns are not sortable or filterable: `Roles`, `External/Internal IP`, `CPU`, `RAM` (logic to determine their value is calculated in the browser) + - `Workloads` list: + - The `Workloads` list, which showed multiple different resource types has been removed. + - Server-Side Pagination of multiple resources is not currently possible. + - `Workloads` group --> All lists + - `Pod Restarts` and `Workload Health` columns have been removed. + - [Re-enable Pod Restart Count and Pod Health columns for Workload lists #14211](https://github.com/rancher/dashboard/issues/14211) + - `Workloads` group / `Job` List + - `Duration` is not sortable (sorting on a duration). + - [Implement more complex server-side pagination sorting #12815](https://github.com/rancher/dashboard/issues/12815) + - `Workloads` group / `Pod` List + - `Images` is not sortable (sorting on an array). + - `Service Discovery` group / `Ingresses` + - `Default` is not sortable/filterable (logic to determine their value is calculated in the browser). + - `Storage` group / `ConfigMaps` + - `Data` is not sortable/filterable (logic to determine their value is calculated in the browser). + - `Storage` group / `Secrets` + - `Data` is not sortable/filterable (logic to determine their value is calculated in the browser). diff --git a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md index 2295d1cf196..63b78e20f4c 100644 --- a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md +++ b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md @@ -49,3 +49,4 @@ Rancher supports several major cloud providers, but by default, these node drive There are several other node drivers that are disabled by default, but are packaged in Rancher: * [Harvester](../../../../integrations-in-rancher/harvester/overview.md#harvester-node-driver/), available as of Rancher v2.6.1 +* [Google GCE](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md), available as of Rancher v2.12.0 diff --git a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito.md b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito.md new file mode 100644 index 00000000000..58a82dfc38d --- /dev/null +++ b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito.md @@ -0,0 +1,60 @@ +--- +title: Configure Amazon Cognito +description: Create an Amazon Cognito user pool and configure Rancher to work with Amazon Cognito. Your users can then sign into Rancher using their login from Amazon Cognito. +--- + + + + + +If your organization uses Amazon Cognito for user authentication, you can configure Rancher to allow login using Amazon Cognito credentials. The following instructions describe how to configure Rancher to work with Amazon Cognito: + +## Prerequisites + +- In Rancher: + - Amazon Cognito is disabled. + +:::note +Consult the Amazon Cognito [documentation](https://aws.amazon.com/cognito/getting-started/) to configure the user pool. +::: + +- In Amazon Cognito: + - Create a new user pool or use an existing one. + - In the `App client` settings, set the redirect URL to `https://yourRancherHostURL/verify-auth`. Replace `yourRancherHostURL` with the actual hostname of your Rancher instance (e.g., https://rancher.example.com/verify-auth). + +## Configuring Amazon Cognito in Rancher + +1. In the upper left corner of the Rancher UI, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. Select **Amazon Cognito**. +1. Complete the **Configure an Amazon Cognito account** form. For help with filling the form, see the [configuration reference](#configuration-reference). +1. Click **Enable**. + + Rancher will redirect you to the Amazon Cognito login page. Enter your Amazon Cognito credentials to validate your Rancher configuration. + + :::note + + You may need to disable your popup blocker to see the Amazon Cognito login page. + + ::: + +**Result:** Rancher is configured to work with your Amazon Cognito using the OIDC protocol. Your users can now sign into Rancher using their Amazon Cognito logins. + +:::note +User and group search is not supported for Amazon Cognito. When assigning permissions to a Project or Cluster, you must manually enter the UserID generated by Cognito +if the user has not yet logged in to Rancher. However, if the user has previously logged in, you can assign permissions using their username or email address. +::: + +## Configuration Reference + +| Field | Description | +| ------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Client ID | The Client ID of your Amazon Cognito App Client. | +| Client Secret | The generated Secret of your Amazon Cognito App Client. | +| Issuer | The Issuer URL of your Amazon Cognito App Client. It follows the format `https://cognito-idp.{region}.amazonaws.com/{userPoolId}`, and can be found in the App Client settings page. Rancher uses the Issuer URL to fetch all of the required URLs. | + +## Troubleshooting + +### You are not redirected to your authentication provider + +If you fill out the **Configure an Amazon Cognito account** form and click on **Enable**, and you are not redirected to Amazon Cognito, verify your Amazon Cognito configuration. diff --git a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md index 22aab5d5cac..8d4c6dd8ad9 100644 --- a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md +++ b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md @@ -196,6 +196,34 @@ As previously mentioned, custom roles can be defined for use at the cluster or p When defining a custom role, you can grant access to specific resources or specify roles from which the custom role should inherit. A custom role can be made up of a combination of specific grants and inherited roles. All grants are additive. This means that defining a narrower grant for a specific resource **will not** override a broader grant defined in a role that the custom role is inheriting from. +#### UpdatePSA For Project Level + +About defining custom roles, you can grant permission to a user to create or update *PSA* policies when defining namespaces within projects. + +To do so, you can use the following `RoleTemplate` to be applied on the cluster: + +```yaml +apiVersion: management.cattle.io/v3 +builtin: false +context: project +description: '' +displayName: Manage PSA Labels +external: false +hidden: false +kind: RoleTemplate +metadata: + name: namespaces-psa +rules: + - apiGroups: + - management.cattle.io + resources: + - projects + verbs: + - updatepsa +``` + +When creating a new project (from the **Members** tab), click **Add** to add the user and select **Custom** > **Create Namespaces** (to allow the user to create namespaces). Then click **Add** again and select `UpdatePSA` project role template from the list of **Project Permissions**. + ### Default Cluster and Project Roles By default, when a standard user creates a new cluster or project, they are automatically assigned an ownership role: either [cluster owner](#cluster-roles) or [project owner](#project-roles). However, in some organizations, these roles may overextend administrative access. In this use case, you can change the default role to something more restrictive, such as a set of individual roles or a custom role. diff --git a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center.md b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center.md new file mode 100644 index 00000000000..0c049f2076b --- /dev/null +++ b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center.md @@ -0,0 +1,23 @@ +--- +title: Notification Center +--- + + + + + +## What is the Notification Center? + +The Notification Center, located in the upper-right corner of your Rancher dashboard and marked by a bell icon, is your central hub for staying informed about various events within Rancher. + +Notifications are categorized by severity and type: + +* **Error** indicates a high-severity issue. +* **Warning** indicates a medium-severity concern. +* **Information** provides general updates of lower severity. +* **Task** shows that a process or action is currently in progress. +* **Success** confirms that a process or action has completed successfully. + +For example, you might receive a notification when a new version of Rancher is available, or a direct link to the Rancher Release Notes. + +You can easily browse through your notifications using the up and down arrow keys. Certain notifications may also include an associated action, allowing you to respond immediately. diff --git a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md index 0ea24911d5a..d6d2942afd2 100644 --- a/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md +++ b/docs/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md @@ -38,9 +38,6 @@ You can assign a PSA template at the same time that you create a downstream clus 1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. 1. Click **Save**. -### Hardening the Cluster - -If you select the **rancher-restricted** template but don't select a **CIS Profile**, you won't meet required CIS benchmarks. See the [RKE2 hardening guide](../../../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md) for more details. diff --git a/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md b/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md index 9d3a6307a5f..2edb0398c8c 100644 --- a/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md +++ b/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md @@ -70,7 +70,7 @@ To perform a backup, a custom resource of type Backup must be created. folder: rancher region: us-west-2 endpoint: s3.us-west-2.amazonaws.com - resourceSetName: rancher-resource-set + resourceSetName: rancher-resource-set-full encryptionConfigSecretName: encryptionconfig schedule: "@every 1h" retentionCount: 10 @@ -78,7 +78,7 @@ To perform a backup, a custom resource of type Backup must be created. :::note - When creating the Backup resource using YAML editor, the `resourceSetName` must be set to `rancher-resource-set` + When creating the Backup resource using YAML editor, the `resourceSetName` must be set to `rancher-resource-set-full` or `rancher-resource-set-basic`. ::: diff --git a/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md b/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md index 71467ac7403..c3b26afd434 100644 --- a/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md +++ b/docs/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md @@ -28,7 +28,7 @@ The `rancher-backup` operator introduces three custom resources: Backups, Restor The ResourceSet defines which Kubernetes resources need to be backed up. The ResourceSet is not available to be configured in the Rancher UI because the values required to back up Rancher are predefined. This ResourceSet should not be modified. -When a Backup custom resource is created, the `rancher-backup` operator calls the `kube-apiserver` to get the resources in the ResourceSet (specifically, the predefined `rancher-resource-set`) that the Backup custom resource refers to. +When a Backup custom resource is created, the `rancher-backup` operator calls the `kube-apiserver` to get the resources in the ResourceSet that the Backup custom resource refers to. The operator then creates the backup file in the .tar.gz format and stores it in the location configured in the Backup resource. diff --git a/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md index 844fbaad30f..f8fdc984308 100644 --- a/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +++ b/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -170,6 +170,13 @@ You can define the default behavior for newly created clusters or existing ones Changes to the global **imported-cluster-version-management** setting take effect during the cluster’s next reconciliation cycle. +:::note + +If version management is enabled for a cluster, Rancher will deploy the system-upgrade-controller app, along with the associated Plans and other required Kubernetes resources, to the cluster. +If version management is disabled, Rancher will remove these components from the cluster. + +::: + ## Configuring RKE2 and K3s Cluster Upgrades :::tip diff --git a/docs/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md b/docs/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md index 89e4a40ba5d..362ffac61e5 100644 --- a/docs/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md +++ b/docs/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md @@ -14,15 +14,16 @@ This page is about secrets in general. For details on setting up a private regis ::: -When configuring a workload, you'll be able to choose which secrets to include. Like config maps, secrets can be referenced by workloads as either an environment variable or a volume mount. +When configuring a workload, you are able to choose which secrets to include. Like config maps, secrets can be referenced by workloads as either an environment variable or a volume mount. -Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) +Mounted secrets are updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) ## Creating Secrets in Namespaces 1. In the upper left corner, click **☰ > Cluster Management**. 1. Go to the cluster where you want to add a secret and click **Explore**. 1. To navigate to secrets, you may click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Select the **Namespaced** tab. 1. Click **Create**. 1. Select the type of secret you want to create. 1. Select a **Namespace** for the secret. @@ -48,32 +49,66 @@ Mounted secrets will be updated automatically unless they are mounted as subpath **Result:** Your secret is added to the namespace you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. -Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) +Mounted secrets are updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) ## Creating Secrets in Projects -Before v2.6, secrets were required to be in a project scope. Projects are no longer required, and you may use the namespace scope instead. As a result, the Rancher UI was updated to reflect this new functionality. However, you may still create project-scoped secrets if desired. Note that you have to first enable the `legacy` feature flag and look at a single project to do so. Use the following steps to set up your project-level secret: +When creating a secret in a project scope, the secret is copied into all namespaces within the project. -1. In the upper left corner, click **☰ > Global Settings** in the dropdown. -1. Click **Feature Flags**. -1. Go to the `legacy` feature flag and click **Activate**. -1. In the upper left corner, click **☰ > Cluster Management** in the dropdown. -1. Go to the cluster that you created and click **Explore.** -1. Click **Legacy > Projects**. -1. In the top navigation bar, filter to see only one project. -1. In the left navigation bar, click **Secrets**. -1. Click **Add Secret**. +### Creating a Project Scoped Secret in the UI -**Result:** Your secret is added to the individual project you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a secret and click **Explore**. +1. To navigate to secrets, you may click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Select the **Project Scoped** tab. +1. Click **Create Project Scoped Secret**. +1. Select the type of secret you want to create. +1. Select a **Project** for the secret. +1. Enter a **Name** for the secret. -Since project-scoped secrets are set at the project level, any changes made at the namespace level will be overwritten. + :::note -:::note + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a namespace can have duplicate names. If you create a project scoped secret that has the same name as an existing secret in one of the project namespaces, the existing secret is overwritten. -Project-scoped secrets on the local cluster are only visible when a single project is selected. + ::: -::: +1. From **Data**, click **Add** to add a key-value pair. Add as many values as you need. + + :::tip + + You can add multiple key-value pairs to the secret by copying and pasting. + + ::: + + ![](/img/bulk-key-values.gif) + +1. Click **Save**. + +**Result:** Your secret is added to each namespace within the project. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. + +### Creating a Project Scoped Secret with kubectl + +Project scoped secrets work by creating the original secret on the management cluster in what's known as the "Project Backing Namespace". Rancher stores important project related information in this namespace. You can find it in the `status.backingNamespace` field in the project CRD, or by doing `kubectl get projects -A` in the management cluster. + +In order for the secret to be acknowledged by Rancher as a project scoped secret, it also needs the label `management.cattle.io/project-scoped-secret: `. + +Example yaml: + +``` +apiVersion: v1 +data: + key: ZG9n +kind: Secret +metadata: + labels: + management.cattle.io/project-scoped-secret: p-vwxyz + name: test-secret + namespace: c-abc123-p-vwxyz +type: Opaque +``` + +In the above YAML, the namespace is the backing namespace of project `p-vwxyz` and the project scoped secret label references the projectID. When applied to the management cluster, all namespaces within the project `p-vwxyz` contain a copy of `test-secret`. ## What's Next? diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md index 66010c77149..c8c2c442025 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md @@ -54,12 +54,18 @@ If you use RKE2 to set up a cluster, your nodes must meet the [requirements](htt ### Launching Kubernetes on New Nodes in an Infrastructure Provider -RKE2 provisioning is built on top of a new provisioning framework that leverages the upstream [Cluster API](https://github.com/kubernetes-sigs/cluster-api) project. With this new provisioning framework, you can: +RKE2 provisioning is built on top of a new provisioning framework that leverages the upstream [Cluster API (CAPI)](https://github.com/kubernetes-sigs/cluster-api) project. With this new provisioning framework, you can: - Provision RKE2 clusters onto any provider for which Rancher has a node driver - Fully configure RKE2 clusters within Rancher - Choose CNI options Calico, Cilium, and Multus in addition to Canal +When you make changes to your cluster configuration in RKE2, this may result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. + +The following are some specific example configuration changes that may cause the described behavior: + +- When editing the cluster and enabling drain before delete, the existing control plane nodes and worker are deleted and new nodes are created. + RKE2 provisioning also includes installing RKE2 on clusters with Windows nodes. Windows features for RKE2 include: diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md deleted file mode 100644 index ed8db366fbc..00000000000 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Behavior Differences Between RKE1 and RKE2 ---- - - - - - - - -RKE2, also known as RKE Government, is a Kubernetes distribution that focuses on security and compliance for U.S. Federal Government entities. It is considered the next iteration of the Rancher Kubernetes Engine, now known as RKE1. - -RKE1 and RKE2 have several slight behavioral differences to note, and this page will highlight some of these at a high level. - -### Control Plane Components - -RKE1 uses Docker for deploying and managing control plane components, and it also uses Docker as the container runtime for Kubernetes. By contrast, RKE2 launches control plane components as static pods that are managed by the kubelet. RKE2's container runtime is containerd, which allows things such as mirroring a container image registry. RKE1 with Docker does not allow mirroring. - -### Cluster API - -RKE2/K3s provisioning is built on top of the Cluster API (CAPI) upstream framework which often makes RKE2-provisioned clusters behave differently than RKE1-provisioned clusters. - -When you make changes to your cluster configuration in RKE2, this **may** result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. - -The following are some specific example configuration changes that may cause the described behavior: - -- When editing the cluster and enabling `drain before delete`, the existing control plane nodes and worker are deleted and new nodes are created. - -Users who are used to RKE1 provisioning should take note of this new RKE2 behavior which may be unexpected. - -### Terminology - -You will notice that some terms have changed or gone away going from RKE1 to RKE2. For example, in RKE1 provisioning, you use **node templates**; in RKE2 provisioning, you can configure your cluster node pools when creating or editing the cluster. Another example is that the term **node pool** in RKE1 is now known as **machine pool** in RKE2. - - - - diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md index 6f64d0abb70..7e6b5b57d11 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md @@ -6,53 +6,11 @@ title: Creating a DigitalOcean Cluster -In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in DigitalOcean. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in DigitalOcean. -First, you will set up your DigitalOcean cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in DigitalOcean. +First, you will set up your DigitalOcean cloud credentials in Rancher. -Then you will create a DigitalOcean cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. - - - - -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - -### 1. Create your cloud credentials - -1. Click **☰ > Cluster Management**. -1. Click **Cloud Credentials**. -1. Click **Create**. -1. Click **DigitalOcean**. -1. Enter your Digital Ocean credentials. -1. Click **Create**. - -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. - -### 2. Create a node template with your cloud credentials - -Creating a [node template](use-new-nodes-in-an-infra-provider.md#node-templates) for DigitalOcean will allow Rancher to provision new nodes in DigitalOcean. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates**. -1. Click **Add Template**. -1. Click **DigitalOcean**. -1. Fill out a node template for DigitalOcean. For help filling out the form, refer to [DigitalOcean Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md) - -### 3. Create a cluster with node pools using the node template - -1. Click **☰ > Cluster Management**. -1. On the **Clusters** page, click **Create**. -1. Click **DigitalOcean**. -1. Enter a **Cluster Name**. -1. Add one or more node pools to your cluster. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](use-new-nodes-in-an-infra-provider.md) -1. **In the Cluster Configuration** section, choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) -1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Click **Create**. - - - +Then you will create a DigitalOcean cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. ### 1. Create your cloud credentials @@ -77,13 +35,10 @@ Use Rancher to create a Kubernetes cluster in DigitalOcean. 1. Enter a **Cluster Name**. 1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. 1. For each machine pool, define the machine configuration. Refer to the [DigitalOcean machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md) for information on configuration options. -1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. 1. Click **Create**. - - - **Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md new file mode 100644 index 00000000000..d7a09f4089c --- /dev/null +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md @@ -0,0 +1,107 @@ +--- +title: Creating a Google Compute Engine cluster +--- + + + + + + +In this section, you'll learn how to use Rancher to provision an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster on the Google Cloud Platform (GCP) using Google Compute Engine (GCE) through Rancher. + + +First, you will enable the GCE node driver in the Rancher UI. Then, you follow the steps to create a GCP service account with the necessary permissions, and generate a JSON key file. This key file will be used to create a cloud credential in Rancher. + + +Then, you will create a GCE cluster in Rancher, and when configuring the cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE2 onto the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. + + +1. [Enable the GCE node driver](#1-enable-the-gce-node-driver) +1. [Create your cloud credential](#2-create-a-cloud-credential) +1. [Create a GCE cluster with your cloud credential](#3-create-a-cluster-using-the-cloud-credential) +1. [GCE Best Practices](#gce-best-practices) + +### Prerequisites + +1. A valid Google Cloud Platform account and project. +1. A GCP Service Account JSON key file. The service account associated with this key must have the following IAM roles: + 1. **Compute Admin** + 1. **Service Account User** + 1. **Viewer** +1. A VPC Network to provision VMs within. + +Refer to the [GCP documentation](https://cloud.google.com/iam/docs/service-account-overview) on creating and managing service account keys for more details. + + +### 1. Enable the GCE node driver + +The GCE node driver is not enabled by default in Rancher. You must enable it before you can provision GCE clusters, or work with GCE specific CRDs. + +1. Click **☰ > Cluster Management**. +1. On the left hand side, click **Drivers**. +1. Open the **Node Drivers** tab. +1. Find the **Google GCE** driver and select **⋮ > Activate**. + + +### 2. Create a cloud credential + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Google**. +1. Enter your GCP Service Account JSON key file. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials in other clusters. Depending on the permissions granted to the service account, this credential may also be used for GKE clusters. + + +### 3. Create a cluster using the cloud credential + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Google GCE**. +1. Select a **Cloud Credential** and provide the GCP project to create the VM in. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to the [Google GCE machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +### GCE Best Practices + +#### External Firewall Rules, Open Ports, and ACE + +If the cluster being provisioned will utilize the [Authorized Cluster Endpoint (ACE) feature](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster), controlplane nodes must expose port `6443`. This port is not exposed in the default machine pool configuration to prevent it from being exposed across all cluster nodes, and to reduce the number of firewall rules created by Rancher. + +In order for ACE to work as expected, you must specify this port in the Rancher UI when configuring the controlplane machine pool by enabling the `Expose external ports` checkbox, under the `Show Advanced` section of the machine pool configuration UI. Alternatively, you may manually create a custom firewall rule in GCP and provide the related network tag in the controlplane machine-pool configuration. + +#### Internal Firewall Rules + +Rancher will automatically create a firewall rule and network tag to facilitate communication between cluster nodes internally within the specified VPC network. This rule will contain the minimum number of ports required to create an RKE2/K3s cluster. + +If you need to extend the number of ports exposed internally between cluster nodes, a new firewall rule should be manually created, and the associated network tag assigned to the relevant machine pools. If desired, the automatic creation of the internal firewall rule can be disabled for each given machine pool when creating or updating the cluster. + +#### Cross Network Deployments + +While it is possible to deploy different machine pools into different VPC networks, the internal firewall rule created by Rancher does not support this configuration by default. To create machine pools in different networks, additional firewall rules to facilitate communication between nodes in different networks must be manually created. + + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md index 3a4db1d0a73..11231c88c5a 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md @@ -7,11 +7,11 @@ description: Learn the prerequisites and steps required in order for you to crea -In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Amazon EC2. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in Amazon EC2. -First, you will set up your EC2 cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in EC2. +First, you will set up your EC2 cloud credentials in Rancher. -Then you will create an EC2 cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. +Then you will create an EC2 cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. ### Prerequisites @@ -26,64 +26,6 @@ Then you will create an EC2 cluster in Rancher, and when configuring the new clu The steps to create a cluster differ based on your Rancher version. - - - -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials and information from EC2](#2-create-a-node-template-with-your-cloud-credentials-and-information-from-ec2) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - -### 1. Create your cloud credentials - -1. Click **☰ > Cluster Management**. -1. Click **Cloud Credentials**. -1. Click **Create**. -1. Click **Amazon**. -1. Enter a name for the cloud credential. -1. In the **Default Region** field, select the AWS region where your cluster nodes will be located. -1. Enter your AWS EC2 **Access Key** and **Secret Key**. -1. Click **Create**. - -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. - -### 2. Create a node template with your cloud credentials and information from EC2 - -Creating a [node template](use-new-nodes-in-an-infra-provider.md#node-templates) for EC2 will allow Rancher to provision new nodes in EC2. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates** -1. Click **Add Template**. -1. Fill out a node template for EC2. For help filling out the form, refer to [EC2 Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md) -1. Click **Create**. - - :::note - - If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. - - ::: - -### 3. Create a cluster with node pools using the node template - -Add one or more node pools to your cluster. For more information about node pools, see [this section.](use-new-nodes-in-an-infra-provider.md) - -1. Click **☰ > Cluster Management**. -1. On the **Clusters** page, click **Create**. -1. Click **Amazon EC2**. -1. Create a node pool for each Kubernetes role. For each node pool, choose a node template that you created. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](use-new-nodes-in-an-infra-provider.md) -1. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. Refer to [Selecting Cloud Providers](../../kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/set-up-cloud-providers.md) to configure the Kubernetes Cloud Provider. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) - - :::note - - If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. - - ::: - -1. Click **Create**. - - - - ### 1. Create your cloud credentials If you already have a set of cloud credentials to use, skip this section. @@ -97,7 +39,7 @@ If you already have a set of cloud credentials to use, skip this section. 1. Enter your AWS EC2 **Access Key** and **Secret Key**. 1. Click **Create**. -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. ### 2. Create your cluster @@ -109,13 +51,10 @@ If you already have a set of cloud credentials to use, skip this section. 1. Enter a **Cluster Name**. 1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. 1. For each machine pool, define the machine configuration. Refer to [the EC2 machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md) for information on configuration options. -1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. 1. Click **Create**. - - - **Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md index 2b9e2e10cdd..186a95f366c 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md @@ -6,15 +6,15 @@ title: Creating an Azure Cluster -In this section, you'll learn how to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Azure through Rancher. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in Azure through Rancher. -First, you will set up your Azure cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in Azure. +First, you will set up your Azure cloud credentials in Rancher. -Then you will create an Azure cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. +Then you will create an Azure cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. :::caution -When the Rancher RKE cluster is running in Azure and has an Azure load balancer in front, the outbound flow will fail. The workaround for this problem is as follows: +When the Rancher RKE2/K3s cluster is running in Azure and has an Azure load balancer in front, the outbound flow will fail. The workaround for this problem is as follows: - Terminate the SSL/TLS on the internal load balancer - Use the L7 load balancer @@ -23,16 +23,16 @@ For more information, refer to the documentation on [Azure load balancer limitat ::: -For more information on configuring the Kubernetes cluster that Rancher will install on the Azure nodes, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +For more information on configuring the Kubernetes cluster that Rancher will install on the Azure nodes, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration references. -For more information on configuring Azure node templates, refer to the [Azure node template configuration reference.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) +For more information on configuring Azure machines, refer to the [Azure machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md). - [Preparation in Azure](#preparation-in-azure) - [Creating an Azure Cluster](#creating-an-azure-cluster) ## Preparation in Azure -Before creating a node template in Rancher using a cloud infrastructure such as Azure, we must configure Rancher to allow the manipulation of resources in an Azure subscription. +Before a cluster can be deployed, we must configure our Azure subscription to allow the manipulation of its resources by a third party, such as Rancher. To do this, we will first create a new Azure **service principal (SP)** in Azure **Active Directory (AD)**, which, in Azure, is an application user who has permission to manage Azure resources. @@ -45,54 +45,10 @@ az ad sp create-for-rbac \ --scopes="/subscriptions/" ``` -The creation of this service principal returns three pieces of identification information, *The application ID, also called the client ID*, and *The client secret*. This information will be used when you create a node template for Azure. +The creation of this service principal returns the **application ID**, also called the **client ID**, and the **client secret**. This information is used when you create your cloud credentials. ## Creating an Azure Cluster - - - -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - -### 1. Create your cloud credentials - -1. Click **☰ > Cluster Management**. -1. Click **Cloud Credentials**. -1. Click **Create**. -1. Click **Azure**. -1. Enter your Azure credentials. -1. Click **Create**. - -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. - -### 2. Create a node template with your cloud credentials - -Creating a [node template](use-new-nodes-in-an-infra-provider.md#node-templates) for Azure will allow Rancher to provision new nodes in Azure. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates**. -1. Click **Add Template**. -1. Click **Azure**. -1. Fill out a node template for Azure. For help filling out the form, refer to [Azure Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) - -### 3. Create a cluster with node pools using the node template - -Use Rancher to create a Kubernetes cluster in Azure. - -1. Click **☰ > Cluster Management**. -1. On the **Clusters** page, click **Create**. -1. Click **Azure**. -1. Enter a **Cluster Name**. -1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices, see [this section.](use-new-nodes-in-an-infra-provider.md) -1. In the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) -1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Click **Create**. - - - - ### 1. Create your cloud credentials If you already have a set of cloud credentials to use, skip this section. @@ -104,7 +60,7 @@ If you already have a set of cloud credentials to use, skip this section. 1. Enter your Azure credentials. 1. Click **Create**. -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. ### 2. Create your cluster @@ -118,13 +74,10 @@ Use Rancher to create a Kubernetes cluster in Azure. 1. Enter a **Cluster Name**. 1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. 1. For each machine pool, define the machine configuration. Refer to the [Azure machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) for information on configuration options. -1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. 1. Click **Create**. - - - **Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md index 8d498ac35e6..0ecdfd2e185 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md @@ -18,4 +18,4 @@ A Nutanix cluster may consist of multiple groups of VMs with distinct properties ## Creating a Nutanix Cluster -In [this section,](provision-kubernetes-clusters-in-aos.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Nutanix AOS. \ No newline at end of file +In [this section,](provision-kubernetes-clusters-in-aos.md) you'll learn how to use Rancher to install an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in Nutanix AOS. \ No newline at end of file diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md index f5e74b760e5..b33c13a7170 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md @@ -6,9 +6,9 @@ title: Creating a VMware vSphere Virtual Machine Template -Creating virtual machines in a repeatable and reliable fashion can often be difficult. VMware vSphere offers the ability to build one VM that can then be converted to a template. The template can then be used to create identically configured VMs. Rancher leverages this capability within node pools to create identical RKE1 and RKE2 nodes. +Creating virtual machines in a repeatable and reliable fashion can often be difficult. VMware vSphere offers the ability to build one VM that can then be converted to a template. The template can then be used to create identically configured VMs. Rancher leverages this capability to create identical RKE/K3s nodes. -In order to leverage the template to create new VMs, Rancher has some [specific requirements](#requirements) that the VM must have pre-installed. After you configure the VM with these requirements, you will next need to [prepare the VM](#preparing-your-vm) before [creating the template](#creating-a-template). Finally, once preparation is complete, the VM can be [converted to a template](#converting-to-a-template) and [moved into a content library](#moving-to-a-content-library), ready for Rancher node pool usage. +In order to leverage the template to create new VMs, Rancher has some [specific requirements](#requirements) that the VM must have pre-installed. After you configure the VM with these requirements, you will next need to [prepare the VM](#preparing-your-vm) before [creating the template](#creating-a-template). Finally, once preparation is complete, the VM can be [converted to a template](#converting-to-a-template) and [moved into a content library](#moving-to-a-content-library). ## Requirements @@ -47,14 +47,6 @@ The list of packages that need to be installed on the template is as follows: * Windows Container Feature * [cloudbase-init](https://cloudbase.it/cloudbase-init/#download) -* [Docker EE](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server#install-docker) - RKE1 Only - -:::note About the configuration for Windows templates varies between RKE1 and RKE2: - -- RKE1 leverages Docker, so any RKE1 templates need to have Docker EE pre-installed as well -- RKE2 does not require Docker EE, and thus it does not need to be installed - -::: ## Creating a Template diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md index 20eb639ad31..143bea5a219 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md @@ -34,7 +34,7 @@ The following steps create a role with the required privileges and then assign i 4. Go to the **Users and Groups** tab. -5. Create a new user. Fill out the form and then click **OK**. Make sure to note the username and password, because you will need it when configuring node templates in Rancher. +5. Create a new user. Fill out the form and then click **OK**. Make sure to note the username and password, because you will need it when creating cloud credentials in Rancher. ![](/img/rancheruser.png) diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md index 0694c461321..88f0599cdcb 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md @@ -6,16 +6,11 @@ title: Provisioning Kubernetes Clusters in VMware vSphere -In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in VMware vSphere. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in VMware vSphere. -First, you will set up your vSphere cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision nodes in vSphere. - -Then you will create a vSphere cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. - -For details on configuring the vSphere node template, refer to the [vSphere node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) - -For details on configuring RKE Kubernetes clusters in Rancher, refer to the [cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) +First, you will set up your vSphere cloud credentials in Rancher. +Then you will create a vSphere cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. - [Preparation in vSphere](#preparation-in-vmware-vsphere) - [Creating a vSphere Cluster](#creating-a-vmware-vsphere-cluster) @@ -24,11 +19,9 @@ For details on configuring RKE Kubernetes clusters in Rancher, refer to the [clu This section describes the requirements for setting up vSphere so that Rancher can provision VMs and clusters. -The node templates are documented and tested with the vSphere Web Services API version 6.5. - ### Create Credentials in VMware vSphere -Before proceeding to create a cluster, you must ensure that you have a vSphere user with sufficient permissions. When you set up a node template, the template will need to use these vSphere credentials. +Before proceeding to create a cluster, you must ensure that you have a vSphere user with sufficient permissions. Refer to this [how-to guide](create-credentials.md) for instructions on how to create a user in vSphere with the required permissions. These steps result in a username and password that you will need to provide to Rancher, which allows Rancher to provision resources in vSphere. @@ -58,10 +51,6 @@ User-data.iso files may have become orphaned upon node deletion due to a vSphere ::: -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - ### 1. Create your cloud credentials 1. Click **☰ > Cluster Management**. @@ -71,32 +60,21 @@ User-data.iso files may have become orphaned upon node deletion due to a vSphere 1. Enter your vSphere credentials. For help, refer to **Account Access** in the [node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) 1. Click **Create**. -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. -### 2. Create a node template with your cloud credentials - -Creating a [node template](../use-new-nodes-in-an-infra-provider.md#node-templates) for vSphere will allow Rancher to provision new nodes in vSphere. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates**. -1. Click **Create**. -1. Click **Add Template**. -1. Click **vSphere**. -1. Fill out a node template for vSphere. For help filling out the form, refer to the vSphere node template [configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md). -1. Click **Create**. - -### 3. Create a cluster with node pools using the node template +### 2. Create your cluster Use Rancher to create a Kubernetes cluster in vSphere. 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, click **Create**. 1. Click **VMware vSphere**. -1. Enter a **Cluster Name** and use your vSphere cloud credentials. Click **Continue**. +1. Enter a **Cluster Name** and use your vSphere cloud credentials. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. +1. Use **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) 1. If you want to dynamically provision persistent storage or other infrastructure later, you will need to enable the vSphere cloud provider by modifying the cluster YAML file. For details, refer to [in-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). -1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to the nodes, see [this section.](../use-new-nodes-in-an-infra-provider.md#node-pools) 1. Review your options to confirm they're correct. Then click **Create**. **Result:** diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md index 78c437eed81..0234eadecf4 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md @@ -10,8 +10,6 @@ In Rancher v2.8.3 and later, you can configure the graceful shutdown of virtual In RKE2/K3s, you can set up graceful shutdown when you create the cluster, or edit the cluster configuration to add it afterward. -In RKE, you can edit node templates to similar results. - :::note Since Rancher can't detect the platform of an imported cluster, you cannot enable graceful shutdown on VMware vSphere clusters you have imported. @@ -20,30 +18,12 @@ Since Rancher can't detect the platform of an imported cluster, you cannot enabl ## Enable Graceful Shutdown During VMware vSphere Cluster Creation - - - In RKE2/K3s, you can configure new VMware vSphere clusters with graceful shutdown for VMs: 1. Click **☰ > Cluster Management**. 1. Click **Create** and select **VMware vSphere** to provision a new cluster. 1. Under **Machine Pools > Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. - - - -In RKE, you can't directly configure a new cluster with graceful shutdown. However, you can configure node templates which automatically create node pools with graceful shutdown enabled. The node template can then be used to provision new VMware vSphere clusters that have a graceful shutdown delay. - -1. Click **☰ > Cluster Management**. -1. From the left navigation, select **RKE1 Configuration > Node Templates**. -1. Click **Add Template** and select **vSphere** to create a node template. -1. Under **2. Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. - -When you [use the newly-created node template to create node pools](../use-new-nodes-in-an-infra-provider.md), the nodes will gracefully shutdown of VMs according to the **Graceful Shutdown Timeout** value you have set. - - - - ## Enable Graceful Shutdown in Existing RKE2/K3s Clusters In RKE2/K3s, you can edit the configuration of an existing VMware vSphere cluster to enable graceful shutdown, which adds a delay before deleting VMs. @@ -52,14 +32,3 @@ In RKE2/K3s, you can edit the configuration of an existing VMware vSphere cluste 1. On the **Clusters** page, find the VMware vSphere hosted cluster you want to edit. Click **⋮** at the end of the row associated with the cluster. Select **Edit Config**. 1. Under **Machine Pools > Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. -## Enable Graceful Shutdown in Existing RKE Clusters - -In RKE, you can't directly edit an existing cluster's configuration to add graceful shutdown to existing VMware vSphere clusters. However, you can edit the configuration of existing node templates. As noted in [Updating a Node Template](../../../../../reference-guides/user-settings/manage-node-templates.md#updating-a-node-template), all node pools using the node template automatically use the updated information when new nodes are added to the cluster. - -To edit an existing node template to enable graceful shutdown: - -1. Click **☰ > Cluster Management**. -1. From the left navigation, select **RKE1 Configuration > Node Templates**. -1. Find the VMware vSphere node template you want to edit. Click **⋮** at the end of the row associated with the template. Select **Edit**. -1. Under **2. Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. -1. Click **Save**. diff --git a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md index 553905b2a0b..f155dd8263d 100644 --- a/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md +++ b/docs/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md @@ -15,33 +15,9 @@ Rancher can provision nodes in vSphere and install Kubernetes on them. When crea A vSphere cluster may consist of multiple groups of VMs with distinct properties, such as the amount of memory or the number of vCPUs. This grouping allows for fine-grained control over the sizing of nodes for each Kubernetes role. -## VMware vSphere Enhancements - -The vSphere node templates allow you to bring cloud operations on-premises with the following enhancements: - -### Self-healing Node Pools - -One of the biggest advantages of provisioning vSphere nodes with Rancher is that it allows you to take advantage of Rancher's self-healing node pools, also called the [node auto-replace feature,](../use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) in your on-premises clusters. Self-healing node pools are designed to help you replace worker nodes for stateless applications. When Rancher provisions nodes from a node template, Rancher can automatically replace unreachable nodes. - -:::caution - -It is not recommended to enable node auto-replace on a node pool of master nodes or nodes with persistent volumes attached, because VMs are treated ephemerally. When a node in a node pool loses connectivity with the cluster, its persistent volumes are destroyed, resulting in data loss for stateful applications. - -::: - -### Dynamically Populated Options for Instances and Scheduling - -Node templates for vSphere have been updated so that when you create a node template with your vSphere credentials, the template is automatically populated with the same options for provisioning VMs that you have access to in the vSphere console. - -For the fields to be populated, your setup needs to fulfill the [prerequisites.](provision-kubernetes-clusters-in-vsphere.md#preparation-in-vmware-vsphere) - -### More Supported Operating Systems - -You can provision VMs with any operating system that supports `cloud-init`. Only YAML format is supported for the [cloud config.](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) - ## Creating a VMware vSphere Cluster -In [this section,](provision-kubernetes-clusters-in-vsphere.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in vSphere. +In [this section,](provision-kubernetes-clusters-in-vsphere.md) you'll learn how to use Rancher to install an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in vSphere. ## Provisioning Storage diff --git a/docs/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/docs/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md index 2c32c5c3660..fb48a8c25fe 100644 --- a/docs/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md +++ b/docs/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -48,14 +48,13 @@ Rancher will discover and show resources created by `kubectl`. However, these re ## Authenticating Directly with a Downstream Cluster -This section intended to help you set up an alternative method to access an [RKE cluster.](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) +This section intended to help you set up an alternative method to access a [Rancher-launched cluster.](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) -This method is only available for RKE, RKE2, and K3s clusters that have the [authorized cluster endpoint](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled. When Rancher creates the cluster, it generates a kubeconfig file that includes additional kubectl context(s) for accessing your cluster. This additional context allows you to use kubectl to authenticate with the downstream cluster without authenticating through Rancher. For a longer explanation of how the authorized cluster endpoint works, refer to [this page](authorized-cluster-endpoint.md). +This method is only available RKE2 and K3s clusters that have the [authorized cluster endpoint](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled. When Rancher creates the cluster, it generates a kubeconfig file that includes additional kubectl context(s) for accessing your cluster. This additional context allows you to use kubectl to authenticate with the downstream cluster without authenticating through Rancher. For a longer explanation of how the authorized cluster endpoint works, refer to [this page](authorized-cluster-endpoint.md). On RKE2 and K3s clusters, you need to [manually enable](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) authorized cluster endpoints. -We recommend that as a best practice, you should set up this method to access your RKE, RKE2, and K3s clusters, so that just in case you can’t connect to Rancher, you can still access the cluster. - +We recommend that as a best practice, you should set up this method to access your RKE2 and K3s clusters, so that just in case you can’t connect to Rancher, you can still access the cluster. :::note Prerequisites: @@ -74,7 +73,7 @@ CURRENT NAME CLUSTER AUTHINFO N In this example, when you use `kubectl` with the first context, `my-cluster`, you will be authenticated through the Rancher server. -With the second context, `my-cluster-controlplane-1`, you would authenticate with the authorized cluster endpoint, communicating with an downstream RKE cluster directly. +With the second context, `my-cluster-controlplane-1`, you would authenticate with the authorized cluster endpoint, communicating with an downstream RKE/K3s cluster directly. We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) diff --git a/docs/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/docs/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md index 2ed1d943849..0bd0562745e 100644 --- a/docs/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md +++ b/docs/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -123,65 +123,6 @@ Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). ## Cleaning up Nodes - - -Before you run the following commands, first remove the node through the Rancher UI. - -To remove a node: - -1. Click **☰** and select **Cluster Management**. -1. In the table of clusters, click the name of the cluster the node belongs to. -1. In the first tab, click the checkbox next to the node's state. -1. Click **Delete**. - -If you remove the entire cluster instead of an individual node, or skip rermoving the node through the Rancher UI, follow these steps: - -1. [Remove](#docker-containers-images-and-volumes) the Docker containers from the node and [unmount](#mounts) any volumes. -1. Reboot the node. -1. [Remove](#directories-and-files) any remaining files. -1. Confirm that network interfaces and IP tables were properly cleaned after the reboot. If not, reboot one more time. - -### Windows Nodes - -To clean up a Windows node, run the script in `c:\etc\rancher`. This script deletes Kubernetes-generated resources and the execution binary. It also drops the firewall rules and network settings: - -``` -pushd c:\etc\rancher -.\cleanup.ps1 -popd -``` - -After you run this script, the node is reset and can be re-added to a Kubernetes cluster. - -### Docker Containers, Images, and Volumes - -:::caution - -Be careful when cleaning up Docker containers. The following command will remove *all* Docker containers, images, and volumes on the node, including non-Rancher related containers: - -::: - -``` -docker rm -f $(docker ps -qa) -docker rmi -f $(docker images -q) -docker volume rm $(docker volume ls -q) -``` - -### Mounts - -Kubernetes components and secrets leave behind the following mounts: - -* `/var/lib/kubelet` -* `/var/lib/rancher` -* Miscellaneous mounts in `/var/lib/kubelet/pods/` - -To unmount all mounts, run: - -``` -for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done -``` - - :::note @@ -248,54 +189,7 @@ Depending on the role you assigned to the node, certain directories may or may n ::: - - - -| Directories | -|------------------------------| -| `/etc/ceph` | -| `/etc/cni` | -| `/etc/kubernetes` | -| `/opt/cni` | -| `/opt/rke` | -| `/run/calico` | -| `/run/flannel` | -| `/run/secrets/kubernetes.io` | -| `/var/lib/calico` | -| `/var/lib/cni` | -| `/var/lib/etcd` | -| `/var/lib/kubelet` | -| `/var/lib/rancher/rke` | -| `/var/lib/weave` | -| `/var/log/containers` | -| `/var/log/kube-audit` | -| `/var/log/pods` | -| `/var/run/calico` | - -**To clean the directories:** - -```shell -rm -rf /etc/ceph \ - /etc/cni \ - /etc/kubernetes \ - /opt/cni \ - /opt/rke \ - /run/calico \ - /run/flannel \ - /run/secrets/kubernetes.io \ - /var/lib/calico \ - /var/lib/cni \ - /var/lib/etcd \ - /var/lib/kubelet \ - /var/lib/rancher/rke \ - /var/lib/weave \ - /var/log/containers \ - /var/log/kube-audit \ - /var/log/pods \ - /var/run/calico -``` - - + | Directories | diff --git a/docs/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/docs/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md index 6f1010ff99c..259fcecbcf6 100644 --- a/docs/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md +++ b/docs/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -16,20 +16,6 @@ By default, Kubernetes clusters require certificates and Rancher launched Kubern Certificates can be rotated for the following services: - - - -- etcd -- kubelet (node certificate) -- kubelet (serving certificate, if [enabled](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)) -- kube-apiserver -- kube-proxy -- kube-scheduler -- kube-controller-manager - - - - - admin - api-server - controller-manager @@ -42,9 +28,6 @@ Certificates can be rotated for the following services: - kubelet - kube-proxy - - - :::note For users who didn't rotate their webhook certificates, and they have expired after one year, please see this [page](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md) for help. @@ -68,15 +51,4 @@ Rancher launched Kubernetes clusters have the ability to rotate the auto-generat ### Additional Notes - - - -Even though the RKE CLI can use custom certificates for the Kubernetes cluster components, Rancher currently doesn't allow the ability to upload these in Rancher launched Kubernetes clusters. - - - - -In RKE2, both etcd and control plane nodes are treated as the same `server` concept. As such, when rotating certificates of services specific to either of these components will result in certificates being rotated on both. The certificates will only change for the specified service, but you will see nodes for both components go into an updating state. You may also see worker only nodes go into an updating state. This is to restart the workers after a certificate change to ensure they get the latest client certs. - - - +In RKE2/K3s, both etcd and control plane nodes are treated as the same `server` concept. As such, when rotating certificates of services specific to either of these components will result in certificates being rotated on both. The certificates will only change for the specified service, but you will see nodes for both components go into an updating state. You may also see worker only nodes go into an updating state. This is to restart the workers after a certificate change to ensure they get the latest client certs. diff --git a/docs/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/docs/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md index 5d56f2c60aa..59d1eed2fac 100644 --- a/docs/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md +++ b/docs/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -6,39 +6,11 @@ title: Encryption Key Rotation -### RKE1 Encryption Key Rotation +:::note Important -1. Enable encryption key rotation with either of the following two options: +Encryption key rotation is enabled by default and cannot be disabled. - - Select the `Enabled` radio button in the Rancher UI under **Cluster Options > Advanced Options > Secrets Encryption**: - - ![Enable Encryption Key Rotation](/img/rke1-enable-secrets-encryption.png) - - - OR, apply the following YAML: - - ```yaml - rancher_kubernetes_engine_config: - services: - kube_api: - secrets_encryption_config: - enabled: true - ``` - -2. Rotate keys in the Rancher UI: - - 2.1. Click **☰ > Cluster Management**. - - 2.2. Select **⋮ > Rotate Encryption Keys** on the far right of the screen next to your chosen cluster: - - ![Encryption Key Rotation](/img/rke1-encryption-key.png) - - - -### RKE2 Encryption Key Rotation - -_**New in v2.6.7**_ - ->**Important:** Encryption key rotation is enabled by default and cannot be disabled. +::: To rotate keys in the Rancher UI: @@ -48,5 +20,4 @@ To rotate keys in the Rancher UI: ![Encryption Key Rotation](/img/rke2-encryption-key.png) - ->**Note:** For more information on RKE2 secrets encryption config, please see the [RKE2 docs](https://docs.rke2.io/security/secrets_encryption). \ No newline at end of file +>**Note:** For more information on RKE2 secrets encryption config, please see the [RKE2 docs](https://docs.rke2.io/security/secrets_encryption). diff --git a/docs/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md b/docs/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md deleted file mode 100644 index 795e64cef29..00000000000 --- a/docs/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Roles-based Access Control ---- - - - - - -This section describes the permissions required to use the rancher-cis-benchmark App. - -The rancher-cis-benchmark is a cluster-admin only feature by default. - -However, the `rancher-cis-benchmark` chart installs these two default `ClusterRoles`: - -- cis-admin -- cis-view - -In Rancher, only cluster owners and global administrators have `cis-admin` access by default. - -Note: If you were using the `cis-edit` role added in Rancher v2.5 setup, it has now been removed since -Rancher v2.5.2 because it essentially is same as `cis-admin`. If you happen to create any clusterrolebindings -for `cis-edit`, please update them to use `cis-admin` ClusterRole instead. - -## Cluster-Admin Access - -Rancher CIS Scans is a cluster-admin only feature by default. -This means only the Rancher global admins, and the cluster’s cluster-owner can: - -- Install/Uninstall the rancher-cis-benchmark App -- See the navigation links for CIS Benchmark CRDs - ClusterScanBenchmarks, ClusterScanProfiles, ClusterScans -- List the default ClusterScanBenchmarks and ClusterScanProfiles -- Create/Edit/Delete new ClusterScanProfiles -- Create/Edit/Delete a new ClusterScan to run the CIS scan on the cluster -- View and Download the ClusterScanReport created after the ClusterScan is complete - - -## Summary of Default Permissions for Kubernetes Default Roles - -The rancher-cis-benchmark creates three `ClusterRoles` and adds the CIS Benchmark CRD access to the following default K8s `ClusterRoles`: - -| ClusterRole created by chart | Default K8s ClusterRole | Permissions given with Role -| ------------------------------| ---------------------------| ---------------------------| -| `cis-admin` | `admin`| Ability to CRUD clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR -| `cis-view` | `view `| Ability to List(R) clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR - - -By default only cluster-owner role will have ability to manage and use `rancher-cis-benchmark` feature. - -The other Rancher roles (cluster-member, project-owner, project-member) do not have any default permissions to manage and use rancher-cis-benchmark resources. - -But if a cluster-owner wants to delegate access to other users, they can do so by creating ClusterRoleBindings between these users and the above CIS ClusterRoles manually. -There is no automatic role aggregation supported for the `rancher-cis-benchmark` ClusterRoles. diff --git a/docs/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md b/docs/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md deleted file mode 100644 index 3920a1588c5..00000000000 --- a/docs/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Skipped and Not Applicable Tests ---- - - - - - -This section lists the tests that are skipped in the permissive test profile for RKE. - -> All the tests that are skipped and not applicable on this page will be counted as Not Applicable in the v2.5 generated report. The skipped test count will only mention the user-defined skipped tests. This allows user-skipped tests to be distinguished from the tests that are skipped by default in the RKE permissive test profile. - -## CIS Benchmark v1.5 - -### CIS Benchmark v1.5 Skipped Tests - -| Number | Description | Reason for Skipping | -| ---------- | ------------- | --------- | -| 1.1.12 | Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) | A system service account is required for etcd data directory ownership. Refer to Rancher's hardening guide for more details on how to configure this ownership. | -| 1.2.6 | Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | -| 1.2.16 | Ensure that the admission control plugin PodSecurityPolicy is set (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 1.2.33 | Ensure that the --encryption-provider-config argument is set as appropriate (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | -| 1.2.34 | Ensure that encryption providers are appropriately configured (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | -| 4.2.6 | Ensure that the --protect-kernel-defaults argument is set to true (Automated) | System level configurations are required before provisioning the cluster in order for this argument to be set to true. | -| 4.2.10 | Ensure that the--tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | -| 5.1.5 | Ensure that default service accounts are not actively used. (Automated) | Kubernetes provides default service accounts to be used. | -| 5.2.2 | Minimize the admission of containers wishing to share the host process ID namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.2.3 | Minimize the admission of containers wishing to share the host IPC namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.2.4 | Minimize the admission of containers wishing to share the host network namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.2.5 | Minimize the admission of containers with allowPrivilegeEscalation (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.3.2 | Ensure that all Namespaces have Network Policies defined (Automated) | Enabling Network Policies can prevent certain applications from communicating with each other. | -| 5.6.4 | The default namespace should not be used (Automated) | Kubernetes provides a default namespace. | - -### CIS Benchmark v1.5 Not Applicable Tests - -| Number | Description | Reason for being not applicable | -| ---------- | ------------- | --------- | -| 1.1.1 | Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | -| 1.1.2 | Ensure that the API server pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | -| 1.1.3 | Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.1.4 | Ensure that the controller manager pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.1.5 | Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.6 | Ensure that the scheduler pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.7 | Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | -| 1.1.8 | Ensure that the etcd pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | -| 1.1.13 | Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | -| 1.1.14 | Ensure that the admin.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | -| 1.1.15 | Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.16 | Ensure that the scheduler.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.17 | Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.1.18 | Ensure that the controller-manager.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.3.6 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | -| 4.1.1 | Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | -| 4.1.2 | Ensure that the kubelet service file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | -| 4.1.9 | Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | -| 4.1.10 | Ensure that the kubelet configuration file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | -| 4.2.12 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | \ No newline at end of file diff --git a/docs/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/docs/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md index 9d6fd0b1876..997aaa48171 100644 --- a/docs/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md +++ b/docs/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -19,10 +19,7 @@ In order to deploy and run the adapter successfully, you need to ensure its vers | Rancher Version | Adapter Version | |-----------------|------------------| -| v2.11.3 | v106.0.0+up6.0.0 | -| v2.11.2 | v106.0.0+up6.0.0 | -| v2.11.1 | v106.0.0+up6.0.0 | -| v2.11.0 | v106.0.0+up6.0.0 | +| v2.12.0 | 107.0.0+up7.0.0 | ### 1. Gain Access to the Local Cluster diff --git a/docs/integrations-in-rancher/cis-scans/cis-scans.md b/docs/integrations-in-rancher/compliance-scans/compliance-scans.md similarity index 86% rename from docs/integrations-in-rancher/cis-scans/cis-scans.md rename to docs/integrations-in-rancher/compliance-scans/compliance-scans.md index f170f997d66..a8b5ede68f0 100644 --- a/docs/integrations-in-rancher/cis-scans/cis-scans.md +++ b/docs/integrations-in-rancher/compliance-scans/compliance-scans.md @@ -1,14 +1,14 @@ --- -title: CIS Scans +title: Compliance Scans --- - + -Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. The CIS scans can run on any Kubernetes cluster, including hosted Kubernetes providers such as EKS, AKS, and GKE. +Rancher can run a security scan to check whether a cluster is deployed according to security best practices as defined in Kubernetes security benchmarks, such as the ones provided by STIG, BSI or CIS. The Compliance scans can run on any Kubernetes cluster, including hosted Kubernetes providers such as EKS, AKS, and GKE. -The `rancher-cis-benchmark` app leverages kube-bench, an open-source tool from Aqua Security, to check clusters for CIS Kubernetes Benchmark compliance. Also, to generate a cluster-wide report, the application utilizes Sonobuoy for report aggregation. +The `rancher-compliance` app leverages kube-bench, an open-source tool from Aqua Security, to check the compliance of clusters against Kubernetes Benchmarks. Also, to generate a cluster-wide report, the application utilizes Sonobuoy for report aggregation. ## About the CIS Benchmark @@ -94,24 +94,22 @@ In order to pass the "Hardened" profile, you will need to follow the steps on th The default profile and the supported CIS benchmark version depends on the type of cluster that will be scanned: -The `rancher-cis-benchmark` supports the CIS 1.6 Benchmark version. +The `rancher-compliance` supports the CIS 1.9 Benchmark version. -- For RKE Kubernetes clusters, the RKE Permissive 1.6 profile is the default. +- For RKE Kubernetes clusters, the RKE Permissive 1.9 profile is the default. - EKS and GKE have their own CIS Benchmarks published by `kube-bench`. The corresponding test profiles are used by default for those clusters. -- For RKE2 Kubernetes clusters, the RKE2 Permissive 1.6 profile is the default. +- For RKE2 Kubernetes clusters, the RKE2 Permissive 1.9 profile is the default. - For cluster types other than RKE, RKE2, EKS and GKE, the Generic CIS 1.5 profile will be used by default. ## About Skipped and Not Applicable Tests -For a list of skipped and not applicable tests, refer to [this page](../../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md). - For now, only user-defined skipped tests are marked as skipped in the generated report. Any skipped tests that are defined as being skipped by one of the default profiles are marked as not applicable. ## Roles-based Access Control -For information about permissions, refer to [this page](rbac-for-cis-scans.md) +For information about permissions, refer to [this page](rbac-for-compliance-scans.md) ## Configuration @@ -119,4 +117,4 @@ For more information about configuring the custom resources for the scans, profi ## How-to Guides -Please refer to the [CIS Scan Guides](../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) to learn how to run CIS scans. +Please refer to the [Compliance Scan Guides](../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) to learn how to run Compliance scans. diff --git a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/configuration-reference.md b/docs/integrations-in-rancher/compliance-scans/configuration-reference.md similarity index 63% rename from versioned_docs/version-2.12/integrations-in-rancher/cis-scans/configuration-reference.md rename to docs/integrations-in-rancher/compliance-scans/configuration-reference.md index 3394bc2702b..4406a091acf 100644 --- a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/configuration-reference.md +++ b/docs/integrations-in-rancher/compliance-scans/configuration-reference.md @@ -3,27 +3,27 @@ title: Configuration --- - + -This configuration reference is intended to help you manage the custom resources created by the `rancher-cis-benchmark` application. These resources are used for performing CIS scans on a cluster, skipping tests, setting the test profile that will be used during a scan, and other customization. +This configuration reference is intended to help you manage the custom resources created by the `rancher-compliance` application. These resources are used for performing compliance scans on a cluster, skipping tests, setting the test profile that will be used during a scan, and other customization. -To configure the custom resources, go to the **Cluster Dashboard** To configure the CIS scans, +To configure the custom resources, go to the **Cluster Dashboard** To configure the compliance scans, 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to configure CIS scans and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark**. +1. On the **Clusters** page, go to the cluster where you want to configure compliance scans and click **Explore**. +1. In the left navigation bar, click **Compliance**. ## Scans -A scan is created to trigger a CIS scan on the cluster based on the defined profile. A report is created after the scan is completed. +A scan is created to trigger a compliance scan on the cluster based on the defined profile. A report is created after the scan is completed. When configuring a scan, you need to define the name of the scan profile that will be used with the `scanProfileName` directive. An example ClusterScan custom resource is below: ```yaml -apiVersion: cis.cattle.io/v1 +apiVersion: compliance.cattle.io/v1 kind: ClusterScan metadata: name: rke-cis @@ -33,11 +33,11 @@ spec: ## Profiles -A profile contains the configuration for the CIS scan, which includes the benchmark version to use and any specific tests to skip in that benchmark. +A profile contains the configuration for the compliance scan, which includes the benchmark version to use and any specific tests to skip in that benchmark. :::caution -By default, a few ClusterScanProfiles are installed as part of the `rancher-cis-benchmark` chart. If a user edits these default benchmarks or profiles, the next chart update will reset them back. So it is advisable for users to not edit the default ClusterScanProfiles. +By default, a few ClusterScanProfiles are installed as part of the `rancher-compliance` chart. If a user edits these default benchmarks or profiles, the next chart update will reset them back. So it is advisable for users to not edit the default ClusterScanProfiles. ::: @@ -50,12 +50,12 @@ When you create a new profile, you will also need to give it a name. An example `ClusterScanProfile` is below: ```yaml -apiVersion: cis.cattle.io/v1 +apiVersion: compliance.cattle.io/v1 kind: ClusterScanProfile metadata: annotations: meta.helm.sh/release-name: clusterscan-operator - meta.helm.sh/release-namespace: cis-operator-system + meta.helm.sh/release-namespace: compliance-operator-system labels: app.kubernetes.io/managed-by: Helm name: "" @@ -70,9 +70,9 @@ spec: A benchmark version is the name of benchmark to run using `kube-bench`, as well as the valid configuration parameters for that benchmark. -A `ClusterScanBenchmark` defines the CIS `BenchmarkVersion` name and test configurations. The `BenchmarkVersion` name is a parameter provided to the `kube-bench` tool. +A `ClusterScanBenchmark` defines the Compliance `BenchmarkVersion` name and test configurations. The `BenchmarkVersion` name is a parameter provided to the `kube-bench` tool. -By default, a few `BenchmarkVersion` names and test configurations are packaged as part of the CIS scan application. When this feature is enabled, these default BenchmarkVersions will be automatically installed and available for users to create a ClusterScanProfile. +By default, a few `BenchmarkVersion` names and test configurations are packaged as part of the Compliance scan application. When this feature is enabled, these default BenchmarkVersions will be automatically installed and available for users to create a ClusterScanProfile. :::caution @@ -89,12 +89,12 @@ A ClusterScanBenchmark consists of the fields: An example `ClusterScanBenchmark` is below: ```yaml -apiVersion: cis.cattle.io/v1 +apiVersion: compliance.cattle.io/v1 kind: ClusterScanBenchmark metadata: annotations: meta.helm.sh/release-name: clusterscan-operator - meta.helm.sh/release-namespace: cis-operator-system + meta.helm.sh/release-namespace: compliance-operator-system creationTimestamp: "2020-08-28T18:18:07Z" generation: 1 labels: @@ -106,4 +106,4 @@ metadata: spec: clusterProvider: "" minKubernetesVersion: 1.15.0 -``` \ No newline at end of file +``` diff --git a/docs/integrations-in-rancher/cis-scans/custom-benchmark.md b/docs/integrations-in-rancher/compliance-scans/custom-benchmark.md similarity index 69% rename from docs/integrations-in-rancher/cis-scans/custom-benchmark.md rename to docs/integrations-in-rancher/compliance-scans/custom-benchmark.md index 4ec353cc60b..fc4bf0e3b4b 100644 --- a/docs/integrations-in-rancher/cis-scans/custom-benchmark.md +++ b/docs/integrations-in-rancher/compliance-scans/custom-benchmark.md @@ -3,19 +3,20 @@ title: Creating a Custom Benchmark Version for Running a Cluster Scan --- - + -Each Benchmark Version defines a set of test configuration files that define the CIS tests to be run by the kube-bench tool. -The `rancher-cis-benchmark` application installs a few default Benchmark Versions which are listed under CIS Benchmark application menu. +Each Benchmark Version defines a set of test configuration files that define the Compliance tests to be run by the kube-bench tool. +The `rancher-compliance` application installs a few default Benchmark Versions which are listed under Compliance application menu. -But there could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. -It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. +But in the following cases, a custom configuration or remediation may be required: -When a cluster scan is run, you need to select a Profile which points to a specific Benchmark Version. +- Non-standard file locations: When Kubernetes binaries, configuration or certificate paths deviate from upstream benchmark defaults. +Example: Unlike traditional Kubernetes, K3s bundles control plane components into a single binary. Therefore,` --anonymous-auth` flag presence and configuration should be verified in K3s' logs (`journalctl`), not via `kube-apiserver` process checks (`ps`). -Follow all the steps below to add a custom Benchmark Version and run a scan using it. +- Alternative risk mitigations: If a setup doesn't meet a check but has an equally effective compensating control with justification. Or simply is not concerned by the check requirement because of its design. +Example: By default, K3s embeds the api server within the k3s process. There is no API server pod specification file, so verifying the latter's file permissions is not required. ## 1. Prepare the Custom Benchmark Version ConfigMap @@ -46,7 +47,7 @@ To prepare a custom benchmark version ConfigMap, suppose we want to add a custom 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark > Benchmark Version**. +1. In the left navigation bar, click **Compliance > Benchmark Version**. 1. Click **Create**. 1. Enter the **Name** and a description for your custom benchmark version. 1. Choose the cluster provider that your benchmark version applies to. @@ -60,7 +61,7 @@ To run a scan using your custom benchmark version, you need to add a new Profile 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark > Profile**. +1. In the left navigation bar, click **Compliance > Profile**. 1. Click **Create**. 1. Provide a **Name** and description. In this example, we name it `foo-profile`. 1. Choose the Benchmark Version from the dropdown. @@ -74,7 +75,7 @@ To run a scan, 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark > Scan**. +1. In the left navigation bar, click **Compliance > Scan**. 1. Click **Create**. 1. Choose the new cluster scan profile. 1. Click **Create**. diff --git a/docs/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans.md b/docs/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans.md new file mode 100644 index 00000000000..71348248436 --- /dev/null +++ b/docs/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans.md @@ -0,0 +1,48 @@ +--- +title: Roles-based Access Control +--- + + + + + +This section describes the permissions required to use the rancher-compliance App. + +The rancher-compliance is a cluster-admin only feature by default. + +However, the `rancher-compliance` chart installs these two default `ClusterRoles`: + +- compliance-admin +- compliance-view + +In Rancher, only cluster owners and global administrators have `compliance-admin` access by default. + +## Cluster-Admin Access + +Rancher Compliance Scans is a cluster-admin only feature by default. +This means only the Rancher global admins, and the cluster’s cluster-owner can: + +- Install/Uninstall the rancher-compliance App +- See the navigation links for Compliance CRDs - ClusterScanBenchmarks, ClusterScanProfiles, ClusterScans +- List the default ClusterScanBenchmarks and ClusterScanProfiles +- Create/Edit/Delete new ClusterScanProfiles +- Create/Edit/Delete a new ClusterScan to run the Compliance scan on the cluster +- View and Download the ClusterScanReport created after the ClusterScan is complete + + +## Summary of Default Permissions for Kubernetes Default Roles + +The rancher-compliance creates three `ClusterRoles` and adds the Compliance CRD access to the following default K8s `ClusterRoles`: + +| ClusterRole created by chart | Default K8s ClusterRole | Permissions given with Role +| ------------------------------| ---------------------------| ---------------------------| +| `compliance-admin` | `admin`| Ability to CRUD clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR +| `compliance-view` | `view `| Ability to List(R) clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR + + +By default only cluster-owner role will have ability to manage and use `rancher-compliance` feature. + +The other Rancher roles (cluster-member, project-owner, project-member) do not have any default permissions to manage and use rancher-compliance resources. + +But if a cluster-owner wants to delegate access to other users, they can do so by creating ClusterRoleBindings between these users and the above Compliance ClusterRoles manually. +There is no automatic role aggregation supported for the `rancher-compliance` ClusterRoles. diff --git a/docs/integrations-in-rancher/harvester/overview.md b/docs/integrations-in-rancher/harvester/overview.md index edd54a6f557..ac94aa363b2 100644 --- a/docs/integrations-in-rancher/harvester/overview.md +++ b/docs/integrations-in-rancher/harvester/overview.md @@ -24,7 +24,7 @@ To navigate to the Harvester cluster, click **☰ > Virtualization Management**. ## Harvester Node Driver -The [Harvester node driver](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/) is generally available for RKE and RKE2 options in Rancher. The node driver is available whether or not the Harvester feature flag is enabled. Note that the node driver is off by default. Users may create RKE or RKE2 clusters on Harvester only from the Cluster Management page. +The [Harvester node driver](https://docs.harvesterhci.io/v1.5/rancher/node/node-driver/) is generally available for K3s and RKE2 options in Rancher. The node driver is available whether or not the Harvester feature flag is enabled. Note that the node driver is off by default. Users may create K3s or RKE2 clusters on Harvester only from the Cluster Management page. Harvester allows `.ISO` images to be uploaded and displayed through the Harvester UI, but this is not supported in the Rancher UI. This is because `.ISO` images usually require additional setup that interferes with a clean deployment (without requiring user intervention), and they are not typically used in cloud environments. @@ -32,11 +32,11 @@ See [Provisioning Drivers](../../how-to-guides/new-user-guides/authentication-pe ## Port Requirements -The port requirements for the Harvester cluster can be found [here](https://docs.harvesterhci.io/v1.1/install/requirements#networking). +The port requirements for the Harvester cluster can be found [here](https://docs.harvesterhci.io/v1.5/install/requirements#networking). In addition, other networking considerations are as follows: - Be sure to enable VLAN trunk ports of the physical switch for VM VLAN networks. -- Follow the networking setup guidance [here](https://docs.harvesterhci.io/v1.1/networking/index). +- Follow the networking setup guidance [here](https://docs.harvesterhci.io/v1.5/networking/index). -For other port requirements for other guest clusters, such as K3s and RKE1, please see [these docs](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters). +For other port requirements for other guest clusters, such as K3s and RKE2, please see [these docs](https://docs.harvesterhci.io/v1.5/install/requirements/#guest-clusters). diff --git a/docs/reference-guides/backup-restore-configuration/backup-configuration.md b/docs/reference-guides/backup-restore-configuration/backup-configuration.md index 054a6de3daa..2ece25be1a0 100644 --- a/docs/reference-guides/backup-restore-configuration/backup-configuration.md +++ b/docs/reference-guides/backup-restore-configuration/backup-configuration.md @@ -33,12 +33,6 @@ While you can create your own ResourceSets to back up custom applications, two R `rancher-resource-set-full` includes all essential secrets in the backup files to ensure Rancher continues running smoothly after a restore or migration. To avoid storing sensitive information in plain text, we strongly advise you to enable encryption with a strong key. -:::note Important: - -`rancher-resource-set` is also included by default with the `rancher-backup` operator. However, this ResourceSet is deprecated and is only being kept for backwards compatibility reasons. `rancher-resource-set` will be removed in Rancher v2.12. Please update your Backup custom resources to use either `rancher-resource-set-full` or `rancher-resource-set-basic`. - -::: - | YAML Directive Name | Description | | ---------------- | ---------------- | | `resourceSetName` | Provide the name of the ResourceSet to define which resources will be included in this backup. | diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md b/docs/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md index c3c9b7a732d..e0b9eb6757f 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md @@ -16,4 +16,4 @@ While a managed cluster is disconnected from Rancher, management operations will - **Cleaning Up Disconnected Clusters**: Regularly remove clusters that will no longer reconnect to Rancher (e.g., clusters that have been decommissioned or destroyed). Keeping such clusters in the Rancher management system consumes unnecessary resources, which could impact Rancher's performance over time. -- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE/RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. +- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index dc200ae62b1..7fda8ae4c35 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -30,9 +30,7 @@ Once you have created these _ClusterOutput_ objects, create a _ClusterFlow_ to c ### Kubernetes Components -_ClusterFlows_ have the ability to collect logs from all containers on all hosts in the Kubernetes cluster. This works well in cases where those containers are part of a Kubernetes pod; however, RKE containers exist outside of the scope of Kubernetes. - -Currently the logs from RKE containers are collected, but are not able to easily be filtered. This is because those logs do not contain information as to the source container (e.g. `etcd` or `kube-apiserver`). +_ClusterFlows_ have the ability to collect logs from all containers on all hosts in the Kubernetes cluster. This works well in cases where those containers are part of a Kubernetes pod. A future release of Rancher will include the source container name which will enable filtering of these component logs. Once that change is made, you will be able to customize a _ClusterFlow_ to retrieve **only** the Kubernetes component logs, and direct them to an appropriate output. diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md index 108689861dd..ffa2096211d 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -98,7 +98,7 @@ Monitoring the availability and performance of all your internal workloads is vi ## Security Monitoring -In addition to monitoring workloads to detect performance, availability or scalability problems, the cluster and the workloads running into it should also be monitored for potential security problems. A good starting point is to frequently run and alert on [CIS Scans](../../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) which check if the cluster is configured according to security best practices. +In addition to monitoring workloads to detect performance, availability or scalability problems, the cluster and the workloads running into it should also be monitored for potential security problems. A good starting point is to frequently run and alert on [Compliance Scans](../../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) which check if the cluster is configured according to security best practices. For the workloads, you can have a look at Kubernetes and Container security solutions like [NeuVector](https://www.suse.com/products/neuvector/), [Falco](https://falco.org/), [Aqua Kubernetes Security](https://www.aquasec.com/solutions/kubernetes-container-security/), [SysDig](https://sysdig.com/). diff --git a/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index c8e12b81efe..877b5084bf9 100644 --- a/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -54,9 +54,6 @@ Consider the following recommendations based on your needs: ### Make sure nodes are configured correctly for Kubernetes It's important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node, checking that all correct ports are opened, and deploying with ssd backed etcd. More details can be found in the [kubernetes docs](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin) and [etcd's performance op guide](https://etcd.io/docs/v3.5/op-guide/performance/). -### When using RKE: Back up the Statefile -RKE keeps record of the cluster state in a file called `cluster.rkestate`. This file is important for the recovery of a cluster and/or the continued maintenance of the cluster through RKE. Because this file contains certificate material, we strongly recommend encrypting this file before backing up. After each run of `rke up` you should backup the state file. - ### Run All Nodes in the Cluster in the Same Datacenter For best performance, run all three of your nodes in the same geographic datacenter. If you are running nodes in the cloud, such as AWS, run each node in a separate Availability Zone. For example, launch node 1 in us-west-2a, node 2 in us-west-2b, and node 3 in us-west-2c. diff --git a/docs/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md b/docs/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md index a760bbebff1..87a74b259c2 100644 --- a/docs/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md +++ b/docs/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md @@ -66,7 +66,7 @@ You should remove any remaining legacy apps that appear in the Cluster Manager U ### Using the Authorized Cluster Endpoint (ACE) -An [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) (ACE) provides access to the Kubernetes API of Rancher-provisioned RKE, RKE2, and K3s clusters. When enabled, the ACE adds a context to kubeconfig files generated for the cluster. The context uses a direct endpoint to the cluster, thereby bypassing Rancher. This reduces load on Rancher for cases where unmediated API access is acceptable or preferable. See [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) for more information and configuration instructions. +An [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) (ACE) provides access to the Kubernetes API of Rancher-provisioned RKE2 and K3s clusters. When enabled, the ACE adds a context to kubeconfig files generated for the cluster. The context uses a direct endpoint to the cluster, thereby bypassing Rancher. This reduces load on Rancher for cases where unmediated API access is acceptable or preferable. See [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) for more information and configuration instructions. ### Reducing Event Handler Executions diff --git a/docs/reference-guides/cluster-configuration/cluster-configuration.md b/docs/reference-guides/cluster-configuration/cluster-configuration.md index 8abd3377435..75bae23a493 100644 --- a/docs/reference-guides/cluster-configuration/cluster-configuration.md +++ b/docs/reference-guides/cluster-configuration/cluster-configuration.md @@ -14,7 +14,6 @@ For information on editing cluster membership, go to [this page.](../../how-to-g The cluster configuration options depend on the type of Kubernetes cluster: -- [RKE Cluster Configuration](rancher-server-configuration/rke1-cluster-configuration.md) - [RKE2 Cluster Configuration](rancher-server-configuration/rke2-cluster-configuration.md) - [K3s Cluster Configuration](rancher-server-configuration/k3s-cluster-configuration.md) - [EKS Cluster Configuration](rancher-server-configuration/eks-cluster-configuration.md) diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md index e0c92a52f0f..b10fe62447f 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md @@ -6,4 +6,4 @@ title: Downstream Cluster Configuration -The following docs will discuss [node template configuration](node-template-configuration/node-template-configuration.md) and [machine configuration](machine-configuration/machine-configuration.md). \ No newline at end of file +The following docs will discuss [machine configuration](machine-configuration/machine-configuration.md). \ No newline at end of file diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md new file mode 100644 index 00000000000..6c320f3b3d4 --- /dev/null +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md @@ -0,0 +1,87 @@ +--- +title: GCE Machine Configuration +--- + + + + + + +For more information about Google Cloud Platform (GCP) and the Google Compute Engine (GCE), refer to the official [GCP documentation](https://cloud.google.com/docs). + +### Zone + +The GCP Region and Zone that the VM will be deployed to. For example, `us-east1-b`. + +### Machine Image Project + +The image project that the desired image families belong to. + +### Machine Image Family + +The image family that the desired machine operating system belongs to. + +### Machine Image + +The operating system that will be installed onto the VM. + +### Disk Type + +The type of the disk attached to the VM. The available types may differ between regions. + +### Disk Size + +The size of the disk attached to the VM, in Gigabytes. + +### Machine Type + +The type of VM that will be deployed. Machine types determine the number of resources (vCPU, RAM, etc.) allocated for each node. + +### Network + +The VPC network that the VM will be created in. This value cannot be changed once the machine pool has been provisioned. + +### Subnet + +The VPC subnetwork tha the VM will be created in. This value cannot be changed once the machine pool has been provisioned. + +### Username + +A custom username set as the default user of the GCE VM. + +### External Address + +The desired external IP address for the GCE VM. + +### Scopes + +A list of OAuth2 scopes which allow the VM to access other GCP APIs. + +### Allow Internal Communication + +By default, a VPC firewall rule is automatically created to expose a fixed set of ports within the VPC to facilitate communication between cluster nodes. This behavior can be disabled on a per machine pool basis, when clicking the `Show Advanced` option and disabling the `Allow Internal Communication` checkbox. + +### Expose External ports + +A list of ports to be opened _externally_ to the wider internet. Open ports are defined at the machine pool level. Enabling this option will result in the automatic creation of a VPC firewall rule. This rule will be automatically deleted when the cluster or machine pool is deleted. + +### Network Tags + +Tags is a list of _network tags_, which can be used to associate preexisting Firewall Rules with all VMs within a machine pool. + +### Labels + +A comma seperated list of custom labels to be attached to all VMs within a given machine pool. Unlike Tags, Labels do not influence networking behavior and only serve to organize cloud resources. + +## Advanced Options + +When creating clusters via the Rancher UI some options are automatically configured for you. However, when creating machine config objects manually, you must ensure you properly configure the below fields. + +### external-firewall-rule-prefix + +A prefix that will be used when creating the firewall rule to expose ports publicly. Ideally, this should be a concatenation the machine pool name and the cluster name. This field must be set if the machine pool is configured to expose ports publicly, otherwise it can be omitted. + +### internal-firewall-rule-prefix + +A prefix that will be used when creating the internal firewall rule which allows for communication between nodes within the cluster. If this field is omitted, no internal firewall rule will be created. + diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md index 0e2aa590833..82be0707c8a 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md @@ -6,7 +6,6 @@ title: Rancher Server Configuration -- [RKE1 Cluster Configuration](rke1-cluster-configuration.md) - [RKE2 Cluster Configuration](rke2-cluster-configuration.md) - [K3s Cluster Configuration](k3s-cluster-configuration.md) - [EKS Cluster Configuration](eks-cluster-configuration.md) diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md index a76cb30552d..44d7d22123b 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md @@ -133,9 +133,9 @@ If the cloud provider you want to use is not listed as an option, you will need The default [pod security admission configuration template](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for the cluster. -##### Worker CIS Profile +##### Worker Compliance Profile -Select a [CIS benchmark](../../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) to validate the system configuration against. +Select a [compliance benchmark](../../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) to validate the system configuration against. ##### Project Network Isolation diff --git a/docs/reference-guides/kubernetes-concepts.md b/docs/reference-guides/kubernetes-concepts.md index e1e880e8cd4..c28d05394ce 100644 --- a/docs/reference-guides/kubernetes-concepts.md +++ b/docs/reference-guides/kubernetes-concepts.md @@ -10,13 +10,7 @@ This page explains concepts related to Kubernetes that are important for underst ## About Docker -Docker is the container packaging and runtime standard. Developers build container images from Dockerfiles and distribute container images from Docker registries. [Docker Hub](https://hub.docker.com) is the most popular public registry. Many organizations also set up private Docker registries. Docker is primarily used to manage containers on individual nodes. - -:::note - -Although Rancher 1.6 supported Docker Swarm clustering technology, it is no longer supported in Rancher 2.x due to the success of Kubernetes. - -::: +Docker is a container packaging and runtime standard. Developers can build container images from Dockerfiles and distribute container images from Docker registries. [Docker Hub](https://hub.docker.com) is a popular public registry. Many organizations also set up private Docker registries. Docker is primarily used to manage containers on individual nodes. ## About Kubernetes @@ -26,7 +20,7 @@ Kubernetes is the container cluster management standard. YAML files specify cont A cluster is a group of computers that work together as a single system. -A _Kubernetes Cluster_ is a cluster that uses the [Kubernetes container-orchestration system](https://kubernetes.io/) to deploy, maintain, and scale Docker containers, allowing your organization to automate application operations. +A _Kubernetes Cluster_ is a cluster that uses the [Kubernetes container-orchestration system](https://kubernetes.io/) to deploy, maintain, and scale containers, allowing your organization to automate application operations. ## Roles for Nodes in Kubernetes Clusters diff --git a/docs/reference-guides/monitoring-v2-configuration/receivers.md b/docs/reference-guides/monitoring-v2-configuration/receivers.md index b1237e3646b..16e3940e735 100644 --- a/docs/reference-guides/monitoring-v2-configuration/receivers.md +++ b/docs/reference-guides/monitoring-v2-configuration/receivers.md @@ -351,29 +351,29 @@ receivers: - service_key: 'database-integration-key' ``` -## Example Route Config for CIS Scan Alerts +## Example Route Config for Compliance Scan Alerts -While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. +While configuring the routes for `rancher-compliance` alerts, you can specify the matching using the key-value pair `job: rancher-compliance-scan`. -For example, the following example route configuration could be used with a Slack receiver named `test-cis`: +For example, the following example route configuration could be used with a Slack receiver named `test-compliance`: ```yaml spec: - receiver: test-cis + receiver: test-compliance group_by: # - string group_wait: 30s group_interval: 30s repeat_interval: 30s match: - job: rancher-cis-scan + job: rancher-compliance-scan # key: string match_re: {} # key: string ``` -For more information on enabling alerting for `rancher-cis-benchmark`, see [this section.](../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) +For more information on enabling alerting for `rancher-compliance-benchmark`, see [this section.](../../how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md) ## Trusted CA for Notifiers diff --git a/docs/reference-guides/rancher-cluster-tools.md b/docs/reference-guides/rancher-cluster-tools.md index ad46fbdd9d2..b89ff4a1299 100644 --- a/docs/reference-guides/rancher-cluster-tools.md +++ b/docs/reference-guides/rancher-cluster-tools.md @@ -42,8 +42,8 @@ Rancher's integration with Istio was improved in Rancher v2.5. For more information, refer to the Istio documentation [here.](../integrations-in-rancher/istio/istio.md) -## CIS Scans +## Compliance Scans -Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. +Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in most recognized Kubernetes Security Benchmarks, such as STIG. -For more information, refer to the CIS scan documentation [here.](../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) \ No newline at end of file +For more information, refer to the Compliance scan documentation [here.](../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) diff --git a/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md index b84375fbda5..99157d3bbba 100644 --- a/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -32,14 +32,6 @@ One option for the underlying Kubernetes cluster is to use K3s Kubernetes. K3s i ![Architecture of a K3s Kubernetes Cluster Running the Rancher Management Server](/img/k3s-server-storage.svg) -### RKE Kubernetes Cluster Installations - -In an RKE installation, the cluster data is replicated on each of three etcd nodes in the cluster, providing redundancy and data duplication in case one of the nodes fails. - -
Architecture of an RKE Kubernetes Cluster Running the Rancher Management Server
- -![Architecture of an RKE Kubernetes cluster running the Rancher management server](/img/rke-server-storage.svg) - ## Recommended Load Balancer Configuration for Kubernetes Installations We recommend the following configurations for the load balancer and Ingress controllers: @@ -61,7 +53,7 @@ For the best performance and greater security, we recommend a dedicated Kubernet ## Recommended Node Roles for Kubernetes Installations -The below recommendations apply when Rancher is installed on a K3s Kubernetes cluster or an RKE Kubernetes cluster. +The below recommendations apply when Rancher is installed on a K3s Kubernetes cluster. ### K3s Cluster Roles @@ -69,38 +61,6 @@ In K3s clusters, there are two types of nodes: server nodes and agent nodes. Bot For the cluster running the Rancher management server, we recommend using two server nodes. Agent nodes are not required. -### RKE Cluster Roles - -If Rancher is installed on an RKE Kubernetes cluster, the cluster should have three nodes, and each node should have all three Kubernetes roles: etcd, controlplane, and worker. - -### Contrasting RKE Cluster Architecture for Rancher Server and for Downstream Kubernetes Clusters - -Our recommendation for RKE node roles on the Rancher server cluster contrasts with our recommendations for the downstream user clusters that run your apps and services. - -Rancher uses RKE as a library when provisioning downstream Kubernetes clusters. Note: The capability to provision downstream K3s clusters will be added in a future version of Rancher. - -For downstream Kubernetes clusters, we recommend that each node in a user cluster should have a single role for stability and scalability. - -![Kubernetes Roles for Nodes in Rancher Server Cluster vs. User Clusters](/img/rancher-architecture-node-roles.svg) - -RKE only requires at least one node with each role and does not require nodes to be restricted to one role. However, for the clusters that run your apps, we recommend separate roles for each node so that workloads on worker nodes don't interfere with the Kubernetes master or cluster data as your services scale. - -We recommend that downstream user clusters should have at least: - -- **Three nodes with only the etcd role** to maintain a quorum if one node is lost, making the state of your cluster highly available -- **Two nodes with only the controlplane role** to make the master component highly available -- **One or more nodes with only the worker role** to run the Kubernetes node components, as well as the workloads for your apps and services - -With that said, it is safe to use all three roles on three nodes when setting up the Rancher server because: - -* It allows one `etcd` node failure. -* It maintains multiple instances of the master components by having multiple `controlplane` nodes. -* No other workloads than Rancher itself should be created on this cluster. - -Because no additional workloads will be deployed on the Rancher server cluster, in most cases it is not necessary to use the same architecture that we recommend for the scalability and reliability of downstream clusters. - -For more best practices for downstream clusters, refer to the [production checklist](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/checklist-for-production-ready-clusters.md) or our [best practices guide.](../best-practices/best-practices.md) - ## Architecture for an Authorized Cluster Endpoint (ACE) If you are using an [authorized cluster endpoint (ACE),](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) we recommend creating an FQDN pointing to a load balancer which balances traffic across your nodes with the `controlplane` role. diff --git a/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index e3dd9cb475e..f42e97da652 100644 --- a/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -41,7 +41,7 @@ There is one cluster controller and one cluster agent for each downstream cluste - Watches for resource changes in the downstream cluster - Brings the current state of the downstream cluster to the desired state - Configures access control policies to clusters and projects -- Provisions clusters by calling the required Docker machine drivers and Kubernetes engines, such as RKE and GKE +- Provisions clusters by calling the required Docker machine drivers and Kubernetes engines, such as GKE By default, to enable Rancher to communicate with a downstream cluster, the cluster controller connects to the cluster agent. If the cluster agent is not available, the cluster controller can connect to a [node agent](#3-node-agents) instead. @@ -62,7 +62,7 @@ The `cattle-node-agent` is deployed using a [DaemonSet](https://kubernetes.io/do An authorized cluster endpoint (ACE) allows users to connect to the Kubernetes API server of a downstream cluster without having to route their requests through the Rancher authentication proxy. -> ACE is available on RKE, RKE2, and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. +> ACE is available on RKE2 and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. There are two main reasons why a user might need the authorized cluster endpoint: @@ -178,11 +178,8 @@ If you see an error related to "impersonation" in the UI, pay close attention to The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster: -- `rancher-cluster.yml`: The RKE cluster configuration file. -- `kube_config_rancher-cluster.yml`: The Kubeconfig file for the cluster, this file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. -- `rancher-cluster.rkestate`: The Kubernetes cluster state file. This file contains credentials for full access to the cluster. Note: This state file is only created when using RKE v0.2.0 or higher. - -> **Note:** The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. +- `config.yaml`: The RKE2 and K3s cluster configuration file. +- `rke2.yaml` or `k3s.yaml`: The Kubeconfig file for your RKE2 or K3s cluster. This file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. For more information on connecting to a cluster without the Rancher authentication proxy and other configuration options, refer to the [kubeconfig file](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) documentation. @@ -194,13 +191,7 @@ The tools that Rancher uses to provision downstream user clusters depends on the Rancher can dynamically provision nodes in a provider such as Amazon EC2, DigitalOcean, Azure, or vSphere, then install Kubernetes on them. -Rancher provisions this type of cluster using [RKE](https://github.com/rancher/rke) and [docker-machine.](https://github.com/rancher/machine) - -### Rancher Launched Kubernetes for Custom Nodes - -When setting up this type of cluster, Rancher installs Kubernetes on existing nodes, which creates a custom cluster. - -Rancher provisions this type of cluster using [RKE.](https://github.com/rancher/rke) +Rancher provisions this type of cluster using [docker-machine.](https://github.com/rancher/machine) ### Hosted Kubernetes Providers diff --git a/docs/reference-guides/rancher-security/hardening-guides/hardening-guides.md b/docs/reference-guides/rancher-security/hardening-guides/hardening-guides.md deleted file mode 100644 index c591b3a2fbd..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/hardening-guides.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Self-Assessment and Hardening Guides for Rancher ---- - - - - - -Rancher provides specific security hardening guides for each supported Rancher version's Kubernetes distributions. - -## Rancher Kubernetes Distributions - -Rancher uses the following Kubernetes distributions: - -- [**RKE**](https://rancher.com/docs/rke/latest/en/), Rancher Kubernetes Engine, is a CNCF-certified Kubernetes distribution that runs entirely within Docker containers. -- [**RKE2**](https://docs.rke2.io/) is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. -- [**K3s**](https://docs.k3s.io/) is a fully conformant, lightweight Kubernetes distribution. It is easy to install, with half the memory requirement of upstream Kubernetes, all in a binary of less than 100 MB. - -To harden a Kubernetes cluster that's running a distribution other than those listed, refer to your Kubernetes provider docs. - -## Hardening Guides and Benchmark Versions - -Each self-assessment guide is accompanied by a hardening guide. These guides were tested alongside the listed Rancher releases. Each self-assessment guides was tested on a specific Kubernetes version and CIS benchmark version. If a CIS benchmark has not been validated for your Kubernetes version, you can use the existing guides until a guide for your version is added. - -### RKE Guides - -| Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | -|--------------------|-----------------------|-----------------------|------------------| -| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke1-hardening-guide/rke1-hardening-guide.md) | - -### RKE2 Guides - -| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | -|------|--------------------|-----------------------|-----------------------|------------------| -| Rancher provisioned RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke2-hardening-guide/rke2-hardening-guide.md) | -| Standalone RKE2 | Kubernetes v1.27-v1.32 | CIS v1.9 | [Link](https://docs.rke2.io/security/cis_self_assessment19) | [Link](https://docs.rke2.io/security/hardening_guide) | - -### K3s Guides - -| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | -|------|--------------------|-----------------------|-----------------------|------------------| -| Rancher provisioned K3s cluster | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](k3s-hardening-guide/k3s-hardening-guide.md) | -| Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment-1.8) | [Link](https://docs.k3s.io/security/hardening-guide) | - -## Rancher with SELinux - -[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a kernel module that adds extra access controls and security tools to Linux. Historically used by government agencies, SELinux is now industry-standard. SELinux is enabled by default on RHEL and CentOS. - -To use Rancher with SELinux, we recommend [installing](../selinux-rpm/about-rancher-selinux.md) the `rancher-selinux` RPM. - diff --git a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide.md b/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide.md deleted file mode 100644 index 5878e17f31b..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide.md +++ /dev/null @@ -1,744 +0,0 @@ ---- -title: K3s Hardening Guides ---- - - - - - -This document provides prescriptive guidance for how to harden a K3s cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. - -:::note -This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. -::: - -This hardening guide is intended to be used for K3s clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | - -:::note -In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter isn't required anymore, and was removed by CIS. -::: - -For more details on how to evaluate a hardened K3s cluster against the official CIS benchmark, refer to the K3s self-assessment guides for specific Kubernetes and CIS benchmark versions. - -K3s passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: - -1. K3s does not modify the host operating system. Any host-level modifications need to be done manually. -2. Certain CIS policy controls for `NetworkPolicies` and `PodSecurityStandards` (`PodSecurityPolicies` on v1.24 and older) restrict cluster functionality. - You must opt into having K3s configure these policies. Add the appropriate options to your command-line flags or configuration file (enable admission plugins), and manually apply the appropriate policies. - See further for more details. - -The first section (1.1) of the CIS Benchmark primarily focuses on pod manifest permissions and ownership. Since everything in the distribution is packaged in a single binary, this section does not apply to the core components of K3s. - -## Host-level Requirements - -### Ensure `protect-kernel-defaults` is set - - - - -The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. - - - - -This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. - -The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. - -```yaml -spec: - rkeConfig: - machineSelectorConfig: - - config: - protect-kernel-defaults: true -``` - - - - -### Set kernel parameters - -The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: - -```ini -vm.panic_on_oom=0 -vm.overcommit_memory=1 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. - -This configuration needs to be done before setting the kubelet flag, otherwise K3s will fail to start. - -## Kubernetes Runtime Requirements - -The CIS Benchmark runtime requirements center around pod security (via PSA), network policies and API Server auditing logs. - -By default, K3s does not include any pod security or network policies. However, K3s ships with a controller that enforces any network policies you create. By default, K3s enables both the `PodSecurity` and `NodeRestriction` admission controllers, among others. - -### Pod Security - - - - -K3s v1.25 and newer support [Pod Security admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) for controlling pod security. - -You can specify the PSA configuration by setting the `defaultPodSecurityAdmissionConfigurationTemplateName` field in the cluster configuration in Rancher: - -```yaml -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted -``` - -The `rancher-restricted` template is provided by Rancher to enforce the highly-restrictive Kubernetes upstream [`Restricted`](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile with best practices for pod hardening. - - - - -K3s v1.24 and older support [Pod Security Policy (PSP)](https://github.com/kubernetes/website/blob/release-1.24/content/en/docs/concepts/security/pod-security-policy.md) for controlling pod security. - -You can enable PSPs by passing the following flags in the cluster configuration in Rancher: - -```yaml -spec: - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount -``` - -This maintains the `NodeRestriction` plugin and enables the `PodSecurityPolicy`. - -Once you enable PSPs, you can apply a policy to satisfy the necessary controls described in section 5.2 of the CIS Benchmark. - -:::note -These are manual checks in the CIS Benchmark. The CIS scan flags the results as `warning`, because manual inspection is necessary by the cluster operator. -::: - -Here is an example of a compliant PSP: - -```yaml ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: restricted-psp -spec: - privileged: false # CIS - 5.2.1 - allowPrivilegeEscalation: false # CIS - 5.2.5 - requiredDropCapabilities: # CIS - 5.2.7/8/9 - - ALL - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false # CIS - 5.2.4 - hostIPC: false # CIS - 5.2.3 - hostPID: false # CIS - 5.2.2 - runAsUser: - rule: 'MustRunAsNonRoot' # CIS - 5.2.6 - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - readOnlyRootFilesystem: false -``` - -For the example PSP to be effective, we need to create a `ClusterRole` and a `ClusterRoleBinding`. We also need to include a "system unrestricted policy" for system-level pods that require additional privileges, and an additional policy that allows the necessary sysctls for full functionality of ServiceLB. - -```yaml ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: restricted-psp -spec: - privileged: false - allowPrivilegeEscalation: false - requiredDropCapabilities: - - ALL - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - rule: 'MustRunAsNonRoot' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - readOnlyRootFilesystem: false ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: system-unrestricted-psp - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' -spec: - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - fsGroup: - rule: RunAsAny - hostIPC: true - hostNetwork: true - hostPID: true - hostPorts: - - max: 65535 - min: 0 - privileged: true - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - '*' ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: svclb-psp - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' -spec: - allowPrivilegeEscalation: false - allowedCapabilities: - - NET_ADMIN - allowedUnsafeSysctls: - - net.ipv4.ip_forward - - net.ipv6.conf.all.forwarding - fsGroup: - rule: RunAsAny - hostPorts: - - max: 65535 - min: 0 - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:restricted-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - restricted-psp ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:system-unrestricted-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svclb-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - svclb-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svc-local-path-provisioner-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svc-coredns-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svc-cis-operator-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: default:restricted-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:restricted-psp -subjects: -- kind: Group - name: system:authenticated - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system-unrestricted-node-psp-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:system-unrestricted-psp -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system-unrestricted-svc-acct-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:system-unrestricted-psp -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svclb-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svclb-psp -subjects: -- kind: ServiceAccount - name: svclb ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svc-local-path-provisioner-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svc-local-path-provisioner-psp -subjects: -- kind: ServiceAccount - name: local-path-provisioner-service-account ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svc-coredns-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svc-coredns-psp -subjects: -- kind: ServiceAccount - name: coredns ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svc-cis-operator-psp-rolebinding - namespace: cis-operator-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svc-cis-operator-psp -subjects: -- kind: ServiceAccount - name: cis-operator-serviceaccount -``` - -The policies presented above can be placed in a file named `policy.yaml` in the `/var/lib/rancher/k3s/server/manifests` directory. Both the policy file and the its directory hierarchy must be created before starting K3s. A restrictive access permission is recommended to avoid leaking potential sensitive information. - -```shell -sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests -``` - -:::note -The critical Kubernetes additions such as CNI, DNS, and Ingress are run as pods in the `kube-system` namespace. Therefore, this namespace has a less restrictive policy, so that these components can run properly. -::: - - - - -### Network Policies - -CIS requires that all namespaces apply a network policy that reasonably limits traffic into namespaces and pods. - -:::note -This is a manual check in the CIS Benchmark. The CIS scan flags the result as a `warning`, because manual inspection is necessary by the cluster operator. -::: - -The network policies can be placed in the `policy.yaml` file in `/var/lib/rancher/k3s/server/manifests` directory. If the directory was not created as part of the PSP (as described above), it must be created first. - -```shell -sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests -``` - -Here is an example of a compliant network policy: - -```yaml ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: kube-system -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: kube-system ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: default -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: default ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: kube-public -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: kube-public -``` - -The active restrictions block DNS unless purposely allowed. Below is a network policy that allows DNS-related traffic: - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-network-dns-policy - namespace: -spec: - ingress: - - ports: - - port: 53 - protocol: TCP - - port: 53 - protocol: UDP - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Ingress -``` - -The metrics-server and Traefik ingress controller are blocked by default if network policies are not created to allow access. - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-metrics-server - namespace: kube-system -spec: - podSelector: - matchLabels: - k8s-app: metrics-server - ingress: - - {} - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-svclbtraefik-ingress - namespace: kube-system -spec: - podSelector: - matchLabels: - svccontroller.k3s.cattle.io/svcname: traefik - ingress: - - {} - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-traefik-v121-ingress - namespace: kube-system -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: traefik - ingress: - - {} - policyTypes: - - Ingress -``` - -:::note -You must manage network policies as normal for any additional namespaces you create. -::: - -### API Server audit configuration - -CIS requirements 1.2.22 to 1.2.25 are related to configuring audit logs for the API Server. K3s does not create by default the log directory and audit policy, as auditing requirements are specific to each user's policies and environment. - -If you need a log directory, it must be created before you start K3s. We recommend a restrictive access permission to avoid leaking sensitive information. - -```bash -sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/logs -``` - -The following is a starter audit policy to log request metadata. This policy should be written to a file named `audit.yaml` in the `/var/lib/rancher/k3s/server` directory. Detailed information about policy configuration for the API server can be found in the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). - -```yaml ---- -apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: Metadata -``` - -Further configurations are also needed to pass CIS checks. These are not configured by default in K3s, because they vary based on your environment and needs: - -- Ensure that the `--audit-log-path` argument is set. -- Ensure that the `--audit-log-maxage` argument is set to 30 or as appropriate. -- Ensure that the `--audit-log-maxbackup` argument is set to 10 or as appropriate. -- Ensure that the `--audit-log-maxsize` argument is set to 100 or as appropriate. - -Combined, to enable and configure audit logs, add the following lines to the K3s cluster configuration file in Rancher: - -```yaml -spec: - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 - - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 - - audit-log-maxage=30 # CIS 1.2.19 - - audit-log-maxbackup=10 # CIS 1.2.20 - - audit-log-maxsize=100 # CIS 1.2.21 -``` - -### Controller Manager Requirements - -CIS requirement 1.3.1 checks for garbage collection settings in the Controller Manager. Garbage collection is important to ensure sufficient resource availability and avoid degraded performance and availability. Based on your system resources and tests, choose an appropriate threshold value to activate garbage collection. - -This can be remediated by setting the following configuration in the K3s cluster file in Rancher. The value below is only an example. The appropriate threshold value is specific to each user's environment. - -```yaml -spec: - rkeConfig: - machineGlobalConfig: - kube-controller-manager-arg: - - terminated-pod-gc-threshold=10 # CIS 1.3.1 -``` - -### Configure `default` Service Account - -Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. - -For CIS requirement 5.1.5 the `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. - -This can be remediated by updating the `automountServiceAccountToken` field to `false` for the `default` service account in each namespace. - -For `default` service accounts in the built-in namespaces (`kube-system`, `kube-public`, `kube-node-lease`, and `default)`, K3s does not automatically do this. - -Save the following configuration to a file called `account_update.yaml`. - -```yaml ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -Create a bash script file called `account_update.sh`. Be sure to `chmod +x account_update.sh` so the script has execute permissions. - -```shell -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -Run the script every time a new service account is added to your cluster. - -## Reference Hardened K3s Template Configuration - -The following reference template configuration is used in Rancher to create a hardened K3s custom cluster based on each CIS control in this guide. This reference does not include other required **cluster configuration** directives, which vary based on your environment. - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted - enableNetworkPolicy: true - kubernetesVersion: # Define K3s version - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - enable-admission-plugins=NodeRestriction,ServiceAccount # CIS 1.2.15, 1.2.13 - - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 - - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 - - audit-log-maxage=30 # CIS 1.2.19 - - audit-log-maxbackup=10 # CIS 1.2.20 - - audit-log-maxsize=100 # CIS 1.2.21 - - request-timeout=300s # CIS 1.2.22 - - service-account-lookup=true # CIS 1.2.24 - kube-controller-manager-arg: - - terminated-pod-gc-threshold=10 # CIS 1.3.1 - secrets-encryption: true - machineSelectorConfig: - - config: - kubelet-arg: - - make-iptables-util-chains=true # CIS 4.2.7 -``` - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - enableNetworkPolicy: true - kubernetesVersion: # Define K3s version - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount # CIS 1.2.15, 5.2, 1.2.13 - - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 - - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 - - audit-log-maxage=30 # CIS 1.2.19 - - audit-log-maxbackup=10 # CIS 1.2.20 - - audit-log-maxsize=100 # CIS 1.2.21 - - request-timeout=300s # CIS 1.2.22 - - service-account-lookup=true # CIS 1.2.24 - kube-controller-manager-arg: - - terminated-pod-gc-threshold=10 # CIS 1.3.1 - secrets-encryption: true - machineSelectorConfig: - - config: - kubelet-arg: - - make-iptables-util-chains=true # CIS 4.2.7 - protect-kernel-defaults: true # CIS 4.2.6 -``` - - - - -## Conclusion - -If you have followed this guide, your K3s custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our K3s self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index fe6b3ef299c..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,3215 +0,0 @@ ---- -title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - -This document is a companion to the [K3s Hardening Guide](k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. - -This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -This document is for Rancher operators, security teams, auditors and decision makers. - -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). - -## Testing Methodology - -Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. - -Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. - -These are the possible results for each control: - -- **Pass** - The K3s cluster passes the audit outlined in the benchmark. -- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. -- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. - -This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. - -:::note - -This guide only covers `automated` (previously called `scored`) tests. - -::: - -### Controls - - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable. - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable. - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable. - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable. - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable. - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable. - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /etc/kubernetes/manifests/etcd.yaml -Not Applicable. - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /etc/kubernetes/manifests/etcd.yaml -Not Applicable. - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 -Not Applicable. - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root -Not Applicable. - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 1.1.11 -``` - -**Expected Result**: - -```console -'700' is equal to '700' -``` - -**Returned Value**: - -```console -700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** Not Applicable - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd -Not Applicable. - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 scheduler - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root scheduler - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 controllermanager - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root controllermanager - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/server/cred/controller.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /var/lib/rancher/k3s/server/tls - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/server/tls -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /etc/kubernetes/pki/*.crt - -**Audit:** - -```bash -stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /etc/kubernetes/pki/*.key - -**Audit:** - -```bash -stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= -Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' is present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... -Permissive - Enabling Pod Security Policy can cause applications to unexpectedly fail. - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log -Permissive. - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 -Permissive. - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 -Permissive. - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 -Permissive. - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s -Permissive. - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 1.2.29 -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= -Permissive - Enabling encryption changes how data can be recovered as data is encrypted. - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. -Permissive - Enabling encryption changes how data can be recovered as data is encrypted. - -### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true -Not Applicable. - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' -``` - -**Expected Result**: - -```console -'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.1 -``` - -**Expected Result**: - -```console -'cert-file' is present AND 'key-file' is present -``` - -**Returned Value**: - -```console -cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key -``` - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and set the below parameter. ---client-cert-auth="true" - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.2 -``` - -**Expected Result**: - -```console -'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -client-cert-auth: true -``` - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.3 -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.4 -``` - -**Expected Result**: - -```console -'cert-file' is present AND 'key-file' is present -``` - -**Returned Value**: - -```console -cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key -``` - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and set the below parameter. ---peer-client-cert-auth=true - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.5 -``` - -**Expected Result**: - -```console -'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -client-cert-auth: true -``` - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.6 -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.7 -``` - -**Expected Result**: - -```console -'trusted-ca-file' is present -``` - -**Returned Value**: - -```console -trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Manual) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - All configuration is passed in as arguments at container run time. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable. - All configuration is passed in as arguments at container run time. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -**Returned Value**: - -```console -600 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit:** - -```bash -stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -**Returned Value**: - -```console -644 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/kubelet/config.yaml - -### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/kubelet/config.yaml -Not Applicable. -All configuration is passed in as arguments at container run time. - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** warn - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' -``` - -**Expected Result**: - -```console -'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable. - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--event-qps' is present OR '--event-qps' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** Not Applicable - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--rotate-certificates' is present OR '--rotate-certificates' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** pass - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable. - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/kubelet/config.yaml -``` - -**Expected Result**: - -```console -'RotateKubeletServerCertificate' is present OR 'RotateKubeletServerCertificate' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** warn - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--pod-max-pids' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** pass - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -**Audit Script:** `check_for_default_sa.sh` - -```bash -#!/bin/bash - -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - -count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) -if [[ ${count_sa} -gt 0 ]]; then - echo "false" - exit -fi - -for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") -do - for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') - do - read kind name <<<$(IFS=","; echo $result) - resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) - if [[ ${resource_count} -gt 0 ]]; then - echo "false" - exit - fi - done -done - - -echo "true" - -``` - -**Audit Execution:** - -```bash -./check_for_default_sa.sh -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Automated) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Automated) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md b/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md deleted file mode 100644 index afa5dc0fef1..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md +++ /dev/null @@ -1,513 +0,0 @@ ---- -title: RKE Hardening Guides ---- - - - - - - - -This document provides prescriptive guidance for how to harden an RKE cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. - -:::note -This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. -::: - -This hardening guide is intended to be used for RKE clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | - -:::note -- In Benchmark v1.24 and later, check id `4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated)` might fail, as `/etc/kubernetes/ssl/kube-ca.pem` is set to 644 by default. -- In Benchmark v1.7, the `--protect-kernel-defaults` (`4.2.6`) parameter isn't required anymore, and was removed by CIS. -::: - -For more details on how to evaluate a hardened RKE cluster against the official CIS benchmark, refer to the RKE self-assessment guides for specific Kubernetes and CIS benchmark versions. - -## Host-level requirements - -### Configure Kernel Runtime Parameters - -The following `sysctl` configuration is recommended for all nodes types in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: - -```ini -vm.overcommit_memory=1 -vm.panic_on_oom=0 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -Run `sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. - -### Configure `etcd` user and group - -A user account and group for the **etcd** service is required to be set up before installing RKE. - -#### Create `etcd` user and group - -To create the **etcd** user and group run the following console commands. -The commands below use `52034` for **uid** and **gid** for example purposes. -Any valid unused **uid** or **gid** could also be used in lieu of `52034`. - -```bash -groupadd --gid 52034 etcd -useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin -``` - -When deploying RKE through its cluster configuration `config.yml` file, update the `uid` and `gid` of the `etcd` user: - -```yaml -services: - etcd: - gid: 52034 - uid: 52034 -``` - -## Kubernetes runtime requirements - -### Configure `default` Service Account - -#### Set `automountServiceAccountToken` to `false` for `default` service accounts - -Kubernetes provides a default service account which is used by cluster workloads where no specific service account is assigned to the pod. -Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. -The default service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. - -For each namespace including `default` and `kube-system` on a standard RKE install, the `default` service account must include this value: - -```yaml -automountServiceAccountToken: false -``` - -Save the following configuration to a file called `account_update.yaml`. - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -Create a bash script file called `account_update.sh`. -Be sure to `chmod +x account_update.sh` so the script has execute permissions. - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. - -### Configure Network Policy - -#### Ensure that all Namespaces have Network Policies defined - -Running different applications on the same Kubernetes cluster creates a risk of one compromised application attacking a neighboring application. Network segmentation is important to ensure that containers can communicate only with those they are supposed to. A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. - -Network Policies are namespace scoped. When a network policy is introduced to a given namespace, all traffic not allowed by the policy is denied. However, if there are no network policies in a namespace all traffic will be allowed into and out of the pods in that namespace. To enforce network policies, a container network interface (CNI) plugin must be enabled. This guide uses [Canal](https://github.com/projectcalico/canal) to provide the policy enforcement. Additional information about CNI providers can be found [here](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/). - -Once a CNI provider is enabled on a cluster a default network policy can be applied. For reference purposes a **permissive** example is provided below. If you want to allow all traffic to all pods in a namespace (even if policies are added that cause some pods to be treated as “isolated”), you can create a policy that explicitly allows all traffic in that namespace. Save the following configuration as `default-allow-all.yaml`. Additional [documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) about network policies can be found on the Kubernetes site. - -:::caution -This network policy is just an example and is not recommended for production use. -::: - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-allow-all -spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress -``` - -Create a bash script file called `apply_networkPolicy_to_all_ns.sh`. Be sure to `chmod +x apply_networkPolicy_to_all_ns.sh` so the script has execute permissions. - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl apply -f default-allow-all.yaml -n ${namespace} -done -``` - -Execute this script to apply the `default-allow-all.yaml` configuration with the **permissive** `NetworkPolicy` to all namespaces. - -## Known Limitations - -- Rancher **exec shell** and **view logs** for pods are **not** functional in a hardened setup when only a public IP is provided when registering custom nodes. This functionality requires a private IP to be provided when registering the custom nodes. - -## Reference Hardened RKE `cluster.yml` Configuration - -The reference `cluster.yml` is used by the RKE CLI that provides the configuration needed to achieve a hardened installation of RKE. RKE [documentation](https://rancher.com/docs/rke/latest/en/installation/) provides additional details about the configuration items. This reference `cluster.yml` does not include the required `nodes` directive which will vary depending on your environment. Documentation for node configuration in RKE can be found [here](https://rancher.com/docs/rke/latest/en/config-options/nodes/). - -The example `cluster.yml` configuration file contains an Admission Configuration policy in the `services.kube-api.admission_configuration` field. This [sample](../../psa-restricted-exemptions.md) policy contains the namespace exemptions necessary for an imported RKE cluster to run properly in Rancher, similar to Rancher's pre-defined [`rancher-restricted`](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) policy. - -If you prefer to use RKE's default `restricted` policy, then leave the `services.kube-api.admission_configuration` field empty and set `services.pod_security_configuration` to `restricted`. See [the RKE docs](https://rke.docs.rancher.com/config-options/services/pod-security-admission) for more information. - - - - -:::note -If you intend to import an RKE cluster into Rancher, please consult the [documentation](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. -::: - -```yaml -# If you intend to deploy Kubernetes in an air-gapped environment, -# please consult the documentation on how to configure custom RKE images. -nodes: [] -kubernetes_version: # Define RKE version -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - # Leave `pod_security_configuration` out if you are setting a - # custom policy in `admission_configuration`. Otherwise set - # it to `restricted` to use RKE's pre-defined restricted policy, - # and remove everything inside `admission_configuration` field. - # - # pod_security_configuration: restricted - # - admission_configuration: - apiVersion: apiserver.config.k8s.io/v1 - kind: AdmissionConfiguration - plugins: - - name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1 - kind: PodSecurityConfiguration - defaults: - enforce: "restricted" - enforce-version: "latest" - audit: "restricted" - audit-version: "latest" - warn: "restricted" - warn-version: "latest" - exemptions: - usernames: [] - runtimeClasses: [] - namespaces: [calico-apiserver, - calico-system, - cattle-alerting, - cattle-csp-adapter-system, - cattle-elemental-system, - cattle-epinio-system, - cattle-externalip-system, - cattle-fleet-local-system, - cattle-fleet-system, - cattle-gatekeeper-system, - cattle-global-data, - cattle-global-nt, - cattle-impersonation-system, - cattle-istio, - cattle-istio-system, - cattle-logging, - cattle-logging-system, - cattle-monitoring-system, - cattle-neuvector-system, - cattle-prometheus, - cattle-provisioning-capi-system, - cattle-resources-system, - cattle-sriov-system, - cattle-system, - cattle-ui-plugin-system, - cattle-windows-gmsa-system, - cert-manager, - cis-operator-system, - fleet-default, - ingress-nginx, - istio-system, - kube-node-lease, - kube-public, - kube-system, - longhorn-system, - rancher-alerting-drivers, - security-scan, - tigera-operator] - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - generate_serving_certificate: true -addons: | - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -```yaml -# If you intend to deploy Kubernetes in an air-gapped environment, -# please consult the documentation on how to configure custom RKE images. -nodes: [] -kubernetes_version: # Define RKE version -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - generate_serving_certificate: true -addons: | - # Upstream Kubernetes restricted PSP policy - # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: restricted-noroot - spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - requiredDropCapabilities: - - ALL - # Allow core volume types. - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - # Require the container to run without root privileges. - rule: 'MustRunAsNonRoot' - seLinux: - # This policy assumes the nodes are using AppArmor rather than SELinux. - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - readOnlyRootFilesystem: false - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: psp:restricted-noroot - rules: - - apiGroups: - - extensions - resourceNames: - - restricted-noroot - resources: - - podsecuritypolicies - verbs: - - use - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: psp:restricted-noroot - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:restricted-noroot - subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:authenticated - --- - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -## Reference Hardened RKE Cluster Template Configuration - -The reference RKE cluster template provides the minimum required configuration to achieve a hardened installation of Kubernetes. RKE templates are used to provision Kubernetes and define Rancher settings. Follow the Rancher [documentation](../../../../getting-started/installation-and-upgrade/installation-and-upgrade.md) for additional information about installing RKE and its template details. - - - - -```yaml -# -# Cluster Config -# -default_pod_security_admission_configuration_template_name: rancher-restricted -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # Define cluster name - -# -# Rancher Config -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # Define RKE version - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: false - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -```yaml -# -# Cluster Config -# -default_pod_security_policy_template_id: restricted-noroot -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # Define cluster name - -# -# Rancher Config -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # Define RKE version - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -## Conclusion - -If you have followed this guide, your RKE custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index ac002a20369..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,2865 +0,0 @@ ---- -title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - - - -This document is a companion to the [RKE Hardening Guide](rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. - - -This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. - -This document is for Rancher operators, security teams, auditors and decision makers. - -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). - -## Testing Methodology - -Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. - -Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. - -:::note - -This guide only covers `automated` (previously called `scored`) tests. - -::: - -### Controls - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit:** - -```bash -stat -c %a /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'700' is equal to '700' -``` - -**Returned Value**: - -```console -700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd - -**Audit:** - -```bash -stat -c %U:%G /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'etcd:etcd' is present -``` - -**Returned Value**: - -```console -etcd:etcd -``` - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /etc/kubernetes/pki/ - -**Audit Script:** `check_files_owner_in_dir.sh` - -```bash -#!/usr/bin/env bash - -# This script is used to ensure the owner is set to root:root for -# the given directory and all the files in it -# -# inputs: -# $1 = /full/path/to/directory -# -# outputs: -# true/false - -INPUT_DIR=$1 - -if [[ "${INPUT_DIR}" == "" ]]; then - echo "false" - exit -fi - -if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then - echo "false" - exit -fi - -statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) -while read -r statInfoLine; do - f=$(echo ${statInfoLine} | cut -d' ' -f1) - p=$(echo ${statInfoLine} | cut -d' ' -f2) - - if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then - if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then - echo "false" - exit - fi - else - if [[ "$p" != "root:root" ]]; then - echo "false" - exit - fi - fi -done <<< "${statInfoLines}" - - -echo "true" -exit - -``` - -**Audit Execution:** - -```bash -./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= -When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'AlwaysPullImages' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-path' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxage' is greater or equal to 30 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxbackup' is greater or equal to 10 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxsize' is greater or equal to 100 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--encryption-provider-config' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. - -**Audit:** - -```bash -ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi -``` - -**Expected Result**: - -```console -'provider' is present -``` - -### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true -Cluster provisioned by RKE handles certificate rotation directly through RKE. - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--cert-file' is present AND '--key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---client-cert-auth="true" - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-cert-file' is present AND '--peer-key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---peer-client-cert-auth=true - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--trusted-ca-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Automated) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. -All configuration is passed in as arguments at container run time. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. - All configuration is passed in as arguments at container run time. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit:** - -```bash -stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit:** - -```bash -stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--event-qps' is greater or equal to 0 OR '--event-qps' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--rotate-certificates' is present OR '--rotate-certificates' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--pod-max-pids' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** pass - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -**Audit Script:** `check_for_default_sa.sh` - -```bash -#!/bin/bash - -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - -count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) -if [[ ${count_sa} -gt 0 ]]; then - echo "false" - exit -fi - -for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") -do - for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') - do - read kind name <<<$(IFS=","; echo $result) - resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) - if [[ ${resource_count} -gt 0 ]]; then - echo "false" - exit - fi - done -done - - -echo "true" - -``` - -**Audit Execution:** - -```bash -./check_for_default_sa.sh -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Manual) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md b/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md deleted file mode 100644 index bf79c65bd58..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md +++ /dev/null @@ -1,274 +0,0 @@ ---- -title: RKE2 Hardening Guides ---- - - - - - -This document provides prescriptive guidance for how to harden an RKE2 cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. - -:::note -This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. -::: - -This hardening guide is intended to be used for RKE2 clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | - -:::note -- In Benchmark v1.24 and later, some check ids might fail due to new file permission requirements (600 instead of 644). Impacted check ids: `1.1.1`, `1.1.3`, `1.1.5`, `1.1.7`, `1.1.13`, `1.1.15`, `1.1.17`, `4.1.3`, `4.1.5` and `4.1.9`. - - In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter is not required anymore, and was removed by CIS. -::: - -For more details on how to evaluate a hardened RKE2 cluster against the official CIS benchmark, refer to the RKE2 self-assessment guides for specific Kubernetes and CIS benchmark versions. - -RKE2 passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: - -1. RKE2 will not modify the host operating system. Therefore, you, the operator, must make a few host-level modifications. -2. Certain CIS controls for Network Policies and Pod Security Standards (or Pod Security Policies (PSP) on RKE2 versions prior to v1.25) will restrict the functionality of the cluster. You must opt into having RKE2 configure these for you. To help ensure these requirements are met, RKE2 can be started with the profile flag set to `cis-1.23` for v1.25 and newer or `cis-1.6` for v1.24 and older. - -## Host-level requirements - -There are two areas of host-level requirements: kernel parameters and etcd process/directory configuration. These are outlined in this section. - -### Ensure `protect-kernel-defaults` is set - - - - -The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. - - - - -This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. - -The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. - -```yaml -spec: - rkeConfig: - machineSelectorConfig: - - config: - protect-kernel-defaults: true -``` - - - - -### Set kernel parameters - -The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: - -```ini -vm.panic_on_oom=0 -vm.overcommit_memory=1 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. - -### Ensure etcd is configured properly - -The CIS Benchmark requires that the etcd data directory be owned by the `etcd` user and group. This implicitly requires the etcd process run as the host-level `etcd` user. To achieve this, RKE2 takes several steps when started with a valid `cis-1.xx` profile: - -1. Check that the `etcd` user and group exists on the host. If they don't, exit with an error. -2. Create etcd's data directory with `etcd` as the user and group owner. -3. Ensure the etcd process is ran as the `etcd` user and group by setting the etcd static pod's `SecurityContext` appropriately. - -To meet the above requirements, you must: - -#### Create the etcd user - -On some Linux distributions, the `useradd` command will not create a group. The `-U` flag is included below to account for that. This flag tells `useradd` to create a group with the same name as the user. - -```bash -sudo useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U -``` - -## Kubernetes runtime requirements - -The runtime requirements to pass the CIS Benchmark are centered around pod security, network policies and kernel parameters. Most of this is automatically handled by RKE2 when using a valid `cis-1.xx` profile, but some additional operator intervention is required. These are outlined in this section. - -### PodSecurity - -RKE2 always runs with some amount of pod security. - - - - -On v1.25 and newer, [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) are used for pod security. - -Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.7-hardened` available in Rancher. - -```yaml -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.23 -``` - -When both the `defaultPodSecurityAdmissionConfigurationTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: - -1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. -2. Applies network policies that allow the cluster to pass associated controls. -3. Configures the Pod Security Admission Controller with the PSA configuration template `rancher-restricted`, to enforce restricted mode in all namespaces, except the ones in the template's exemption list. - These namespaces are exempted to allow system pods to run without restrictions, which is required for proper operation of the cluster. - -:::note -If you intend to import an RKE cluster into Rancher, please consult the [documentation](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. -::: - - - - - -On Kubernetes v1.24 and older, the `PodSecurityPolicy` admission controller is always enabled. - -Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. - -:::note -In the following example the profile is set to `cis-1.6` which is the value defined in the upstream RKE2, but the cluster is actually configured to pass the CIS v1.23 hardened profile -::: - -```yaml -spec: - defaultPodSecurityPolicyTemplateName: restricted-noroot - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.6 -``` - - -When both the `defaultPodSecurityPolicyTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: - -1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. -2. Applies network policies that allow the cluster to pass associated controls. -3. Configures runtime pod security policies that allow the cluster to pass associated controls. - - - - -:::note -The Kubernetes control plane components and critical additions such as CNI, DNS, and Ingress are ran as pods in the `kube-system` namespace. Therefore, this namespace will have a policy that is less restrictive so that these components can run properly. -::: - -### NetworkPolicies - -When ran with a valid `cis-1.xx` profile, RKE2 will put `NetworkPolicies` in place that passes the CIS Benchmark for Kubernetes' built-in namespaces. These namespaces are: `kube-system`, `kube-public`, `kube-node-lease`, and `default`. - -The `NetworkPolicy` used will only allow pods within the same namespace to talk to each other. The notable exception to this is that it allows DNS requests to be resolved. - -:::note -Operators must manage network policies as normal for additional namespaces that are created. -::: - -### Configure `default` service account - -**Set `automountServiceAccountToken` to `false` for `default` service accounts** - -Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. The `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. - -For each namespace including `default` and `kube-system` on a standard RKE2 install, the `default` service account must include this value: - -```yaml -automountServiceAccountToken: false -``` - -For namespaces created by the cluster operator, the following script and configuration file can be used to configure the `default` service account. - -The configuration bellow must be saved to a file called `account_update.yaml`. - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -Create a bash script file called `account_update.sh`. Be sure to `sudo chmod +x account_update.sh` so the script has execute permissions. - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - echo -n "Patching namespace $namespace - " - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. - -### API Server audit configuration - -CIS requirements 1.2.19 to 1.2.22 are related to configuring audit logs for the API Server. When RKE2 is started with the `profile` flag set, it will automatically configure hardened `--audit-log-` parameters in the API Server to pass those CIS checks. - -RKE2's default audit policy is configured to not log requests in the API Server. This is done to allow cluster operators flexibility to customize an audit policy that suits their auditing requirements and needs, as these are specific to each users' environment and policies. - -A default audit policy is created by RKE2 when started with the `profile` flag set. The policy is defined in `/etc/rancher/rke2/audit-policy.yaml`. - -```yaml -apiVersion: audit.k8s.io/v1 -kind: Policy -metadata: - creationTimestamp: null -rules: -- level: None -``` - -## Reference Hardened RKE2 Template Configuration - -The reference template configuration is used in Rancher to create a hardened RKE2 custom cluster. This reference does not include other required **cluster configuration** directives which will vary depending on your environment. - - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted - kubernetesVersion: # Define RKE2 version - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.23 -``` - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - defaultPodSecurityPolicyTemplateName: restricted-noroot - kubernetesVersion: # Define RKE2 version - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.6 - protect-kernel-defaults: true -``` - - - - -## Conclusion - -If you have followed this guide, your RKE2 custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE2 self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index c93773a62a7..00000000000 --- a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,2967 +0,0 @@ ---- -title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - -This document is a companion to the [RKE2 Hardening Guide](rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. - -This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. - -This document is for Rancher operators, security teams, auditors and decision makers. - -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). - -## Testing Methodology - -RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. - -Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. - -:::note - -This guide only covers `automated` (previously called `scored`) tests. - -::: - -### Controls - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then find /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml -name '*etcd*' | xargs stat -c permissions=%a; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 - -**Audit:** - -```bash -ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=644 -``` - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root - -**Audit:** - -```bash -ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root root:root -``` - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd -``` - -**Expected Result**: - -```console -permissions has permissions 700, expected 700 or more restrictive -``` - -**Returned Value**: - -```console -permissions=700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** Not Applicable - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/admin.conf - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'600' is present -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /var/lib/rancher/rke2/server/cred/controller.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'600' is present -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /etc/kubernetes/pki/ - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/tls -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /var/lib/rancher/rke2/server/tls/*.crt - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-certificate-authority' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'AlwaysPullImages' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1018768 2419 99 16:17 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-path' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxage' is greater or equal to 30 -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxbackup' is greater or equal to 10 -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--request-timeout' is not present OR '--request-timeout' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-key-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--encryption-provider-config' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. - -**Audit:** - -```bash -ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi -``` - -**Expected Result**: - -```console -'provider' is present -``` - -### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 -Not Applicable. - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= -Not Applicable. - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and set the below parameter. ---client-cert-auth="true" -Not Applicable. - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= -Not Applicable. - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and set the below parameter. ---peer-client-cert-auth=true -Not Applicable. - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Audit Config:** - -```bash -cat /var/lib/rancher/rke2/server/db/etcd/config -``` - -**Expected Result**: - -```console -'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Automated) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not applicable. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit Script:** `check_cafile_permissions.sh` - -```bash -#!/usr/bin/env bash - -CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') -CAFILE=/node$CAFILE -if test -z $CAFILE; then CAFILE=$kubeletcafile; fi -if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi - -``` - -**Audit Execution:** - -```bash -./check_cafile_permissions.sh -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit Script:** `check_cafile_ownership.sh` - -```bash -#!/usr/bin/env bash - -CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') -CAFILE=/node$CAFILE -if test -z $CAFILE; then CAFILE=$kubeletcafile; fi -if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi - -``` - -**Audit Execution:** - -```bash -./check_cafile_ownership.sh -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable. - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.eventRecordQPS}' is present OR '{.eventRecordQPS}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** pass - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** warn - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{range .tlsCipherSuites[:]}{}{','}{end}' is present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.podPidsLimit}' is present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** warn - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Manual) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Automated) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/docs/reference-guides/rancher-security/rancher-security-best-practices.md b/docs/reference-guides/rancher-security/rancher-security-best-practices.md index fa958639c1f..beeb2888880 100644 --- a/docs/reference-guides/rancher-security/rancher-security-best-practices.md +++ b/docs/reference-guides/rancher-security/rancher-security-best-practices.md @@ -25,6 +25,6 @@ If you require such features, combine Layer 7 firewalls with [external authentic You should protect the following ports behind an [external load balancer](../../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-4-load-balancer) that has SSL offload enabled: - **K3s:** Port 6443, used by the Kubernetes API. -- **RKE and RKE2:** Port 6443, used by the Kubernetes API, and port 9345, used for node registration. +- **RKE2:** Port 6443, used by the Kubernetes API, and port 9345, used for node registration. These ports have TLS SAN certificates which list nodes' public IP addresses. An attacker could use that information to gain unauthorized access or monitor activity on the cluster. Protecting these ports helps mitigate against nodes' public IP addresses being disclosed to potential attackers. diff --git a/docs/reference-guides/rancher-security/rancher-security.md b/docs/reference-guides/rancher-security/rancher-security.md index f16699b8ac6..5d9ece208e7 100644 --- a/docs/reference-guides/rancher-security/rancher-security.md +++ b/docs/reference-guides/rancher-security/rancher-security.md @@ -31,22 +31,14 @@ On this page, we provide security related documentation along with resources to NeuVector is an open-source, container-focused security application that is now integrated into Rancher. NeuVector provides production security, DevOps vulnerability protection, and a container firewall, et al. Please see the [Rancher docs](../../integrations-in-rancher/neuvector/neuvector.md) and the [NeuVector docs](https://open-docs.neuvector.com/) for more information. -## Running a CIS Security Scan on a Kubernetes Cluster +## Running a Compliance Security Scan on a Kubernetes Cluster -Rancher leverages [kube-bench](https://github.com/aquasecurity/kube-bench) to run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the [CIS](https://www.cisecurity.org/cis-benchmarks/) (Center for Internet Security) Kubernetes Benchmark. +Rancher leverages [kube-bench](https://github.com/aquasecurity/kube-bench) to run a security scan to check whether Kubernetes is deployed according to security best practices. -The CIS Kubernetes Benchmark is a reference document that can be used to establish a secure configuration baseline for Kubernetes. - -The Center for Internet Security (CIS) is a 501(c\)(3) non-profit organization, formed in October 2000, with a mission to "identify, develop, validate, promote, and sustain best practice solutions for cyber defense and build and lead communities to enable an environment of trust in cyberspace". - -CIS Benchmarks are best practices for the secure configuration of a target system. CIS Benchmarks are developed through the generous volunteer efforts of subject matter experts, technology vendors, public and private community members, and the CIS Benchmark Development team. - -The Benchmark provides recommendations of two types: Automated and Manual. We run tests related to only Automated recommendations. - -When Rancher runs a CIS security scan on a cluster, it generates a report showing the results of each test, including a summary with the number of passed, skipped and failed tests. The report also includes remediation steps for any failed tests. - -For details, refer to the section on [security scans](../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md). +When Rancher runs a Compliance scan on a cluster, it generates a report showing the results of each test, including a summary with the number of passed, skipped and failed tests. The report also includes remediation steps for any failed tests. +For details, refer to the section on [security scans](../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md). +` ## SELinux RPM [Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. @@ -67,7 +59,7 @@ Each version of the hardening guide is intended to be used with specific version The benchmark self-assessment is a companion to the Rancher security hardening guide. While the hardening guide shows you how to harden the cluster, the benchmark guide is meant to help you evaluate the level of security of the hardened cluster. -Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. This guide will walk through the various controls and provide updated example commands to audit compliance in Rancher created clusters. The original benchmark documents can be downloaded from the [CIS website](https://www.cisecurity.org/benchmark/kubernetes/). +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. The original benchmark documents can be downloaded from the [CIS website](https://www.cisecurity.org/benchmark/kubernetes/). Each version of Rancher's self-assessment guide corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark. diff --git a/docs/reference-guides/rancher-security/security-advisories-and-cves.md b/docs/reference-guides/rancher-security/security-advisories-and-cves.md index 4ae6006a3e9..63388df86e8 100644 --- a/docs/reference-guides/rancher-security/security-advisories-and-cves.md +++ b/docs/reference-guides/rancher-security/security-advisories-and-cves.md @@ -44,7 +44,7 @@ Rancher is committed to informing the community of security issues in our produc | [CVE-2022-31247](https://github.com/rancher/rancher/security/advisories/GHSA-6x34-89p7-95wg) | An issue was discovered in Rancher versions up to and including 2.5.15 and 2.6.6 where a flaw with authorization logic allows privilege escalation in downstream clusters through cluster role template binding (CRTB) and project role template binding (PRTB). The vulnerability can be exploited by any user who has permissions to create/edit CRTB or PRTB (such as `cluster-owner`, `manage cluster members`, `project-owner`, and `manage project members`) to gain owner permission in another project in the same cluster or in another project on a different downstream cluster. | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | | [CVE-2021-36783](https://github.com/rancher/rancher/security/advisories/GHSA-8w87-58w6-hfv8) | It was discovered that in Rancher versions up to and including 2.5.12 and 2.6.3, there is a failure to properly sanitize credentials in cluster template answers. This failure can lead to plaintext storage and exposure of credentials, passwords, and API tokens. The exposed credentials are visible in Rancher to authenticated `Cluster Owners`, `Cluster Members`, `Project Owners`, and `Project Members` on the endpoints `/v1/management.cattle.io.clusters`, `/v3/clusters`, and `/k8s/clusters/local/apis/management.cattle.io/v3/clusters`. | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | | [CVE-2021-36782](https://github.com/rancher/rancher/security/advisories/GHSA-g7j7-h4q8-8w2f) | An issue was discovered in Rancher versions up to and including 2.5.15 and 2.6.6 where sensitive fields like passwords, API keys, and Rancher's service account token (used to provision clusters) were stored in plaintext directly on Kubernetes objects like `Clusters` (e.g., `cluster.management.cattle.io`). Anyone with read access to those objects in the Kubernetes API could retrieve the plaintext version of those sensitive data. The issue was partially found and reported by Florian Struck (from [Continum AG](https://www.continum.net/)) and [Marco Stuurman](https://github.com/fe-ax) (from [Shock Media B.V.](https://www.shockmedia.nl/)). | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | -| [CVE-2022-21951](https://github.com/rancher/rancher/security/advisories/GHSA-vrph-m5jj-c46c) | This vulnerability only affects customers using [Weave](../../faq/container-network-interface-providers.md#weave) Container Network Interface (CNI) when configured through [RKE templates](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/about-rke1-templates.md). A vulnerability was discovered in Rancher versions 2.5.0 up to and including 2.5.13, and 2.6.0 up to and including 2.6.4, where a user interface (UI) issue with RKE templates does not include a value for the Weave password when Weave is chosen as the CNI. If a cluster is created based on the mentioned template, and Weave is configured as the CNI, no password will be created for [network encryption](https://github.com/weaveworks/weave/blob/master/site/tasks/manage/security-untrusted-networks.md) in Weave; therefore, network traffic in the cluster will be sent unencrypted. | 24 May 2022 | [Rancher v2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) and [Rancher v2.5.14](https://github.com/rancher/rancher/releases/tag/v2.5.14) | +| [CVE-2022-21951](https://github.com/rancher/rancher/security/advisories/GHSA-vrph-m5jj-c46c) | This vulnerability only affects customers using Weave Container Network Interface (CNI) when configured through [RKE templates](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/about-rke1-templates.md). A vulnerability was discovered in Rancher versions 2.5.0 up to and including 2.5.13, and 2.6.0 up to and including 2.6.4, where a user interface (UI) issue with RKE templates does not include a value for the Weave password when Weave is chosen as the CNI. If a cluster is created based on the mentioned template, and Weave is configured as the CNI, no password will be created for [network encryption](https://github.com/weaveworks/weave/blob/master/site/tasks/manage/security-untrusted-networks.md) in Weave; therefore, network traffic in the cluster will be sent unencrypted. | 24 May 2022 | [Rancher v2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) and [Rancher v2.5.14](https://github.com/rancher/rancher/releases/tag/v2.5.14) | | [CVE-2021-36784](https://github.com/rancher/rancher/security/advisories/GHSA-jwvr-vv7p-gpwq) | A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3 which allows users who have create or update permissions on [Global Roles](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/manage-role-based-access-control-rbac.md) to escalate their permissions, or those of another user, to admin-level permissions. Global Roles grant users Rancher-wide permissions, such as the ability to create clusters. In the identified versions of Rancher, when users are given permission to edit or create Global Roles, they are not restricted to only granting permissions which they already posses. This vulnerability affects customers who utilize non-admin users that are able to create or edit Global Roles. The most common use case for this scenario is the `restricted-admin` role. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | | [CVE-2021-4200](https://github.com/rancher/rancher/security/advisories/GHSA-hx8w-ghh8-r4xf) | This vulnerability only affects customers using the `restricted-admin` role in Rancher. A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3 where the `global-data` role in `cattle-global-data` namespace grants write access to the Catalogs. Since each user with any level of catalog access was bound to the `global-data` role, this grants write access to templates (`CatalogTemplates`) and template versions (`CatalogTemplateVersions`) for any user with any level of catalog access. New users created in Rancher are by default assigned to the `user` role (standard user), which is not designed to grant write catalog access. This vulnerability effectively elevates the privilege of any user to write access for the catalog template and catalog template version resources. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | | [GHSA-wm2r-rp98-8pmh](https://github.com/rancher/rancher/security/advisories/GHSA-wm2r-rp98-8pmh) | This vulnerability only affects customers using [Continuous Delivery with Fleet](../../integrations-in-rancher/fleet/fleet.md) for continuous delivery with authenticated Git and/or Helm repositories. An issue was discovered in `go-getter` library in versions prior to [`v1.5.11`](https://github.com/hashicorp/go-getter/releases/tag/v1.5.11) that exposes SSH private keys in base64 format due to a failure in redacting such information from error messages. The vulnerable version of this library is used in Rancher through Fleet in versions of Fleet prior to [`v0.3.9`](https://github.com/rancher/fleet/releases/tag/v0.3.9). This issue affects Rancher versions 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3. The issue was found and reported by Dagan Henderson from Raft Engineering. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | diff --git a/docs/reference-guides/rancher-webhook.md b/docs/reference-guides/rancher-webhook.md index 760e51bd32d..220be0f17eb 100644 --- a/docs/reference-guides/rancher-webhook.md +++ b/docs/reference-guides/rancher-webhook.md @@ -20,10 +20,7 @@ Each Rancher version is designed to be compatible with a single version of the w | Rancher Version | Webhook Version | Availability in Prime | Availability in Community | |-----------------|-----------------|-----------------------|---------------------------| -| v2.11.3 | v0.7.3 | ✓ | ✓ | -| v2.11.2 | v0.7.2 | ✓ | ✓ | -| v2.11.1 | v0.7.1 | ✓ | ✓ | -| v2.11.0 | v0.7.0 | ✗ | ✓ | +| v2.12.0 | v0.8.0 | ✗ | ✓ | ## Why Do We Need It? diff --git a/docs/reference-guides/single-node-rancher-in-docker/advanced-options.md b/docs/reference-guides/single-node-rancher-in-docker/advanced-options.md index c4dcde046d9..18cadd15b4c 100644 --- a/docs/reference-guides/single-node-rancher-in-docker/advanced-options.md +++ b/docs/reference-guides/single-node-rancher-in-docker/advanced-options.md @@ -44,6 +44,7 @@ Privileged access is [required.](../../getting-started/installation-and-upgrade/ docker run -d --restart=unless-stopped \ -p 80:80 -p 443:443 \ -v /var/log/rancher/auditlog:/var/log/auditlog \ + -e AUDIT_LOG_ENABLED=true \ -e AUDIT_LEVEL=1 \ --privileged \ rancher/rancher:latest diff --git a/docs/shared-files/_cluster-capabilities-table.md b/docs/shared-files/_cluster-capabilities-table.md index e53e3471aad..c4807fa5dba 100644 --- a/docs/shared-files/_cluster-capabilities-table.md +++ b/docs/shared-files/_cluster-capabilities-table.md @@ -8,7 +8,7 @@ | [Managing Projects, Namespaces and Workloads](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) | ✓ | ✓ | ✓ | ✓ | | [Using App Catalogs](../how-to-guides/new-user-guides/helm-charts-in-rancher/helm-charts-in-rancher.md) | ✓ | ✓ | ✓ | ✓ | | Configuring Tools ([Alerts, Notifiers, Monitoring](../integrations-in-rancher/monitoring-and-alerting/monitoring-and-alerting.md), [Logging](../integrations-in-rancher/logging/logging.md), [Istio](../integrations-in-rancher/istio/istio.md)) | ✓ | ✓ | ✓ | ✓ | -| [Running Security Scans](../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | +| [Running Security Scans](../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | | [Ability to rotate certificates](../how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md) | ✓ | ✓ | | | | Ability to [backup](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md) and [restore](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Rancher-launched clusters | ✓ | ✓ | | ✓4 | | [Cleaning Kubernetes components when clusters are no longer reachable from Rancher](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) | ✓ | | | | diff --git a/docs/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md b/docs/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md index adecdecde12..24fba32d46b 100644 --- a/docs/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md +++ b/docs/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md @@ -23,4 +23,4 @@ A Rancher Admin should now be able to view Rancher audit logs and follow through ## Feature Description - When Kubernetes Audit logs are enabled on the downstream cluster, in each event that is logged, the external Identity Provider's username is now logged for each request, at the "metadata" level. -- When Rancher API Audit logs are enabled on the Rancher installation, the external Identity Provider's username is also logged now at the `auditLog.level=1` for each request that hits the Rancher API server, including the login requests. +- When you enable Rancher API Audit logs for a Rancher installation, the external Identity Provider's username is also logged now at the `auditLog.level=0` for each request that hits the Rancher API server, including login requests. diff --git a/docusaurus.config.js b/docusaurus.config.js index af0f1fc690c..02f83f2df05 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -3,41 +3,41 @@ const tailwindPlugin = require('./plugins/tailwind-plugin.cjs'); module.exports = { - title: 'Rancher', - tagline: '', - url: 'https://ranchermanager.docs.rancher.com/', - baseUrl: '/', - onBrokenLinks: 'warn', - onBrokenMarkdownLinks: 'warn', - favicon: 'img/favicon.png', - organizationName: 'rancher', // Usually your GitHub org/user name. - projectName: 'rancher-docs', // Usually your repo name. + title: "Rancher", + tagline: "", + url: "https://ranchermanager.docs.rancher.com/", + baseUrl: "/", + onBrokenLinks: "warn", + onBrokenMarkdownLinks: "warn", + favicon: "img/favicon.png", + organizationName: "rancher", // Usually your GitHub org/user name. + projectName: "rancher-docs", // Usually your repo name. trailingSlash: false, i18n: { - defaultLocale: 'en', - locales: ['en', 'zh'], + defaultLocale: "en", + locales: ["en", "zh"], localeConfigs: { en: { - label: 'English', + label: "English", }, zh: { - label: '简体中文', + label: "简体中文", }, }, }, webpack: { jsLoader: (isServer) => ({ - loader: require.resolve('swc-loader'), + loader: require.resolve("swc-loader"), options: { jsc: { parser: { - syntax: 'typescript', + syntax: "typescript", tsx: true, }, - target: 'es2017', + target: "es2017", }, module: { - type: isServer ? 'commonjs' : 'es6', + type: isServer ? "commonjs" : "es6", }, }, }), @@ -50,12 +50,12 @@ module.exports = { }, algolia: { // The application ID provided by Algolia - appId: '30NEY6C9UY', + appId: "30NEY6C9UY", // Public API key: it is safe to commit it - apiKey: '8df59222c0ad79fdacb4d45d11e21d2e', + apiKey: "8df59222c0ad79fdacb4d45d11e21d2e", - indexName: 'docs_ranchermanager_rancher_io', + indexName: "docs_ranchermanager_rancher_io", // Optional: see doc section below contextualSearch: true, @@ -64,1485 +64,1514 @@ module.exports = { searchParameters: {}, // Optional: path for search page that enabled by default (`false` to disable it) - searchPagePath: 'search', + searchPagePath: "search", //... other Algolia params }, colorMode: { // 'light' | 'dark' - defaultMode: 'light', + defaultMode: "light", // Hides the switch in the navbar // Useful if you want to support a single color mode disableSwitch: true, }, prism: { - additionalLanguages: ['rust'], + additionalLanguages: ["rust"], }, navbar: { - title: '', + title: "", logo: { - alt: 'logo', - src: 'img/rancher-logo-horiz-color.svg', + alt: "logo", + src: "img/rancher-logo-horiz-color.svg", // href: 'en', }, items: [ { - type: 'docsVersionDropdown', - position: 'left', - dropdownItemsAfter: [{to: '/versions', label: 'All versions'}], + type: "docsVersionDropdown", + position: "left", + dropdownItemsAfter: [{ to: "/versions", label: "All versions" }], dropdownActiveClassDisabled: false, }, { - type: 'localeDropdown', - position: 'left', + type: "localeDropdown", + position: "left", }, { - type: 'search', - position: 'left', + type: "search", + position: "left", }, { - type: 'dropdown', - label: 'Quick Links', - position: 'right', + type: "dropdown", + label: "Quick Links", + position: "right", items: [ { - href: 'https://github.com/rancher/rancher', - label: 'GitHub', + href: "https://github.com/rancher/rancher", + label: "GitHub", }, { - href: 'https://github.com/rancher/rancher-docs', - label: 'Docs GitHub', + href: "https://github.com/rancher/rancher-docs", + label: "Docs GitHub", }, - ] + ], }, { - type: 'dropdown', - label: 'More from SUSE', - position: 'right', + type: "dropdown", + label: "More from SUSE", + position: "right", items: [ { - href: 'https://www.rancher.com', - label: 'Rancher', - className: 'navbar__icon navbar__rancher' + href: "https://www.rancher.com", + label: "Rancher", + className: "navbar__icon navbar__rancher", }, { - type: 'html', + type: "html", value: '
', }, { - href: 'https://elemental.docs.rancher.com/', - label: 'Elemental', - className: 'navbar__icon navbar__elemental' + href: "https://elemental.docs.rancher.com/", + label: "Elemental", + className: "navbar__icon navbar__elemental", }, { - href: 'https://fleet.rancher.io/', - label: 'Fleet', - className: 'navbar__icon navbar__fleet' + href: "https://fleet.rancher.io/", + label: "Fleet", + className: "navbar__icon navbar__fleet", }, { - href: 'https://harvesterhci.io', - label: 'Harvester', - className: 'navbar__icon navbar__harvester' + href: "https://harvesterhci.io", + label: "Harvester", + className: "navbar__icon navbar__harvester", }, { - href: 'https://rancherdesktop.io/', - label: 'Rancher Desktop', - className: 'navbar__icon navbar__rancher__desktop' + href: "https://rancherdesktop.io/", + label: "Rancher Desktop", + className: "navbar__icon navbar__rancher__desktop", }, { - type: 'html', + type: "html", value: '
', }, { - href: 'https://opensource.suse.com', - label: 'More Projects...', - className: 'navbar__icon navbar__suse' + href: "https://opensource.suse.com", + label: "More Projects...", + className: "navbar__icon navbar__suse", }, - ] - } + ], + }, ], }, footer: { - style: 'dark', + style: "dark", links: [], copyright: `Copyright © ${new Date().getFullYear()} SUSE Rancher. All Rights Reserved.`, }, }, presets: [ [ - '@docusaurus/preset-classic', + "@docusaurus/preset-classic", { docs: { - routeBasePath: '/', // Serve the docs at the site's root + routeBasePath: "/", // Serve the docs at the site's root /* other docs plugin options */ - sidebarPath: require.resolve('./sidebars.js'), + sidebarPath: require.resolve("./sidebars.js"), showLastUpdateTime: true, - editUrl: 'https://github.com/rancher/rancher-docs/edit/main/', - lastVersion: 'current', + editUrl: "https://github.com/rancher/rancher-docs/edit/main/", + lastVersion: "current", versions: { current: { - label: 'Latest', + label: "Latest", }, '2.12': { - label: 'v2.12 (Preview)', - path: 'v2.12', - banner: 'unreleased' + label: "v2.12", + path: "v2.12", + banner: "none" }, - '2.11': { - label: 'v2.11', - path: 'v2.11', - banner: 'none' + 2.11: { + label: "v2.11", + path: "v2.11", + banner: "none", }, - '2.10': { - label: 'v2.10', - path: 'v2.10', - banner: 'none' + "2.10": { + label: "v2.10", + path: "v2.10", + banner: "none", }, 2.9: { - label: 'v2.9', - path: 'v2.9', - banner: 'none' + label: "v2.9", + path: "v2.9", + banner: "none", }, 2.8: { - label: 'v2.8', - path: 'v2.8', - className: 'toArchive' + label: "v2.8", + path: "v2.8", + className: "toArchive", }, 2.7: { - label: 'v2.7 (Archived)', - path: 'v2.7', + label: "v2.7 (Archived)", + path: "v2.7", banner: `none`, - noIndex: true + noIndex: true, }, 2.6: { - label: 'v2.6 (Archived)', - path: 'v2.6', + label: "v2.6 (Archived)", + path: "v2.6", banner: `none`, - noIndex: true + noIndex: true, }, 2.5: { - label: 'v2.5 (Archived)', - path: 'v2.5', + label: "v2.5 (Archived)", + path: "v2.5", banner: `none`, - noIndex: true + noIndex: true, }, - '2.0-2.4': { - label: 'v2.0-v2.4 (Archived)', - path: 'v2.0-v2.4', - banner: 'none', - noIndex: true + "2.0-2.4": { + label: "v2.0-v2.4 (Archived)", + path: "v2.0-v2.4", + banner: "none", + noIndex: true, }, }, }, blog: false, // Optional: disable the blog plugin // ... theme: { - customCss: [require.resolve('./src/css/custom.css')], + customCss: [require.resolve("./src/css/custom.css")], }, googleTagManager: { - containerId: 'GTM-57KS2MW', + containerId: "GTM-57KS2MW", }, }, ], [ - 'redocusaurus', + "redocusaurus", { // Plugin Options for loading OpenAPI files specs: [ { - id: 'rancher-api-v2-11', - spec: 'openapi/swagger-v2.11.json', + id: "rancher-api-v2-12", + spec: "openapi/swagger-v2.12.json", // route: '/api/', }, { - id: 'rancher-api-v2-10', - spec: 'openapi/swagger-v2.10.json', + id: "rancher-api-v2-11", + spec: "openapi/swagger-v2.11.json", // route: '/api/', }, { - id: 'rancher-api-v2-9', - spec: 'openapi/swagger-v2.9.json', + id: "rancher-api-v2-10", + spec: "openapi/swagger-v2.10.json", // route: '/api/', }, { - id: 'rancher-api-v2-8', - spec: 'openapi/swagger-v2.8.json', + id: "rancher-api-v2-9", + spec: "openapi/swagger-v2.9.json", + // route: '/api/', + }, + { + id: "rancher-api-v2-8", + spec: "openapi/swagger-v2.8.json", // route: '/api/', }, ], // Theme Options for modifying how redoc renders them theme: { // Change with your site colors - primaryColor: '#1890ff', + primaryColor: "#1890ff", }, }, ], ], plugins: [ - tailwindPlugin, - [ - '@docusaurus/plugin-client-redirects', + tailwindPlugin, + [ + "@docusaurus/plugin-client-redirects", { - fromExtensions: ['html', 'htm'], + fromExtensions: ["html", "htm"], redirects: [ - { // Redirects for multi-cluster apps removal (rancher-docs/issues/734) - to: '/integrations-in-rancher/fleet', - from: ['/pages-for-subheaders/deploy-apps-across-clusters', '/how-to-guides/new-user-guides/deploy-apps-across-clusters', '/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet', '/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps'] + { + // Redirects for multi-cluster apps removal (rancher-docs/issues/734) + to: "/integrations-in-rancher/fleet", + from: [ + "/pages-for-subheaders/deploy-apps-across-clusters", + "/how-to-guides/new-user-guides/deploy-apps-across-clusters", + "/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet", + "/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps", + ], }, { - to: '/v2.8/integrations-in-rancher/fleet', - from: ['/v2.8/pages-for-subheaders/deploy-apps-across-clusters', '/v2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters', '/v2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet', '/v2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps'] - },// Redirects for multi-cluster apps removal (rancher-docs/issues/734) (end) + to: "/v2.8/integrations-in-rancher/fleet", + from: [ + "/v2.8/pages-for-subheaders/deploy-apps-across-clusters", + "/v2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters", + "/v2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet", + "/v2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps", + ], + }, // Redirects for multi-cluster apps removal (rancher-docs/issues/734) (end) { - to: '/faq/deprecated-features', - from: '/faq/deprecated-features-in-v2.5' + to: "/faq/deprecated-features", + from: "/faq/deprecated-features-in-v2.5", }, { - to: '/v2.8/faq/deprecated-features', - from: '/v2.8/faq/deprecated-features-in-v2.5' + to: "/v2.8/faq/deprecated-features", + from: "/v2.8/faq/deprecated-features-in-v2.5", }, - { // Redirects for pages-for-subheaders removal [2.8] - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers', - from: '/v2.8/pages-for-subheaders/about-provisioning-drivers' + { + // Redirects for pages-for-subheaders removal [2.8] + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers", + from: "/v2.8/pages-for-subheaders/about-provisioning-drivers", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates', - from: '/v2.8/pages-for-subheaders/about-rke1-templates' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates", + from: "/v2.8/pages-for-subheaders/about-rke1-templates", }, { - to: '/v2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters', - from: '/v2.8/pages-for-subheaders/access-clusters' + to: "/v2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters", + from: "/v2.8/pages-for-subheaders/access-clusters", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration', - from: '/v2.8/pages-for-subheaders/advanced-configuration' + to: "/v2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration", + from: "/v2.8/pages-for-subheaders/advanced-configuration", }, { - to: '/v2.8/how-to-guides/advanced-user-guides', - from: '/v2.8/pages-for-subheaders/advanced-user-guides' + to: "/v2.8/how-to-guides/advanced-user-guides", + from: "/v2.8/pages-for-subheaders/advanced-user-guides", }, { - to: '/v2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install', - from: '/v2.8/pages-for-subheaders/air-gapped-helm-cli-install' + to: "/v2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install", + from: "/v2.8/pages-for-subheaders/air-gapped-helm-cli-install", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config', - from: '/v2.8/pages-for-subheaders/authentication-config' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config", + from: "/v2.8/pages-for-subheaders/authentication-config", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration', - from: '/v2.8/pages-for-subheaders/authentication-permissions-and-global-configuration' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration", + from: "/v2.8/pages-for-subheaders/authentication-permissions-and-global-configuration", }, { - to: '/v2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace', - from: '/v2.8/pages-for-subheaders/aws-cloud-marketplace' + to: "/v2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace", + from: "/v2.8/pages-for-subheaders/aws-cloud-marketplace", }, { - to: '/v2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery', - from: '/v2.8/pages-for-subheaders/backup-restore-and-disaster-recovery' + to: "/v2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery", + from: "/v2.8/pages-for-subheaders/backup-restore-and-disaster-recovery", }, { - to: '/v2.8/reference-guides/backup-restore-configuration', - from: '/v2.8/pages-for-subheaders/backup-restore-configuration' + to: "/v2.8/reference-guides/backup-restore-configuration", + from: "/v2.8/pages-for-subheaders/backup-restore-configuration", }, { - to: '/v2.8/reference-guides/best-practices', - from: '/v2.8/pages-for-subheaders/best-practices' + to: "/v2.8/reference-guides/best-practices", + from: "/v2.8/pages-for-subheaders/best-practices", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters', - from: '/v2.8/pages-for-subheaders/checklist-for-production-ready-clusters' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters", + from: "/v2.8/pages-for-subheaders/checklist-for-production-ready-clusters", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/cis-scan-guides', - from: '/v2.8/pages-for-subheaders/cis-scan-guides' + to: "/v2.8/how-to-guides/advanced-user-guides/cis-scan-guides", + from: "/v2.8/pages-for-subheaders/cis-scan-guides", }, { - to: '/v2.8/integrations-in-rancher/cis-scans', - from: '/v2.8/pages-for-subheaders/cis-scans' + to: "/v2.8/integrations-in-rancher/cis-scans", + from: "/v2.8/pages-for-subheaders/cis-scans", }, { - to: '/v2.8/reference-guides/cli-with-rancher', - from: '/v2.8/pages-for-subheaders/cli-with-rancher' + to: "/v2.8/reference-guides/cli-with-rancher", + from: "/v2.8/pages-for-subheaders/cli-with-rancher", }, { - to: '/v2.8/integrations-in-rancher/cloud-marketplace', - from: '/v2.8/pages-for-subheaders/cloud-marketplace' + to: "/v2.8/integrations-in-rancher/cloud-marketplace", + from: "/v2.8/pages-for-subheaders/cloud-marketplace", }, { - to: '/v2.8/reference-guides/cluster-configuration', - from: '/v2.8/pages-for-subheaders/cluster-configuration' + to: "/v2.8/reference-guides/cluster-configuration", + from: "/v2.8/pages-for-subheaders/cluster-configuration", }, { - to: '/v2.8/integrations-in-rancher/istio/configuration-options', - from: '/v2.8/pages-for-subheaders/configuration-options' + to: "/v2.8/integrations-in-rancher/istio/configuration-options", + from: "/v2.8/pages-for-subheaders/configuration-options", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml', - from: '/v2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml", + from: "/v2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap', - from: '/v2.8/pages-for-subheaders/configure-openldap' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap", + from: "/v2.8/pages-for-subheaders/configure-openldap", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml', - from: '/v2.8/pages-for-subheaders/configure-shibboleth-saml' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml", + from: "/v2.8/pages-for-subheaders/configure-shibboleth-saml", }, { - to: '/v2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage', - from: '/v2.8/pages-for-subheaders/create-kubernetes-persistent-storage' + to: "/v2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage", + from: "/v2.8/pages-for-subheaders/create-kubernetes-persistent-storage", }, { - to: '/v2.8/integrations-in-rancher/logging/custom-resource-configuration', - from: '/v2.8/pages-for-subheaders/custom-resource-configuration' + to: "/v2.8/integrations-in-rancher/logging/custom-resource-configuration", + from: "/v2.8/pages-for-subheaders/custom-resource-configuration", }, { - to: '/v2.8/getting-started/quick-start-guides/deploy-rancher-manager', - from: '/v2.8/pages-for-subheaders/deploy-rancher-manager' + to: "/v2.8/getting-started/quick-start-guides/deploy-rancher-manager", + from: "/v2.8/pages-for-subheaders/deploy-rancher-manager", }, { - to: '/v2.8/getting-started/quick-start-guides/deploy-workloads', - from: '/v2.8/pages-for-subheaders/deploy-rancher-workloads' + to: "/v2.8/getting-started/quick-start-guides/deploy-workloads", + from: "/v2.8/pages-for-subheaders/deploy-rancher-workloads", }, { - to: '/v2.8/reference-guides/cluster-configuration/downstream-cluster-configuration', - from: '/v2.8/pages-for-subheaders/downstream-cluster-configuration' + to: "/v2.8/reference-guides/cluster-configuration/downstream-cluster-configuration", + from: "/v2.8/pages-for-subheaders/downstream-cluster-configuration", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/enable-experimental-features', - from: '/v2.8/pages-for-subheaders/enable-experimental-features' + to: "/v2.8/how-to-guides/advanced-user-guides/enable-experimental-features", + from: "/v2.8/pages-for-subheaders/enable-experimental-features", }, { - to: '/v2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration', - from: '/v2.8/pages-for-subheaders/gke-cluster-configuration' + to: "/v2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration", + from: "/v2.8/pages-for-subheaders/gke-cluster-configuration", }, { - to: '/v2.8/how-to-guides/new-user-guides/helm-charts-in-rancher', - from: '/v2.8/pages-for-subheaders/helm-charts-in-rancher' + to: "/v2.8/how-to-guides/new-user-guides/helm-charts-in-rancher", + from: "/v2.8/pages-for-subheaders/helm-charts-in-rancher", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler', - from: '/v2.8/pages-for-subheaders/horizontal-pod-autoscaler' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler", + from: "/v2.8/pages-for-subheaders/horizontal-pod-autoscaler", }, { - to: '/v2.8/how-to-guides/new-user-guides/infrastructure-setup', - from: '/v2.8/pages-for-subheaders/infrastructure-setup' + to: "/v2.8/how-to-guides/new-user-guides/infrastructure-setup", + from: "/v2.8/pages-for-subheaders/infrastructure-setup", }, { - to: '/v2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler', - from: '/v2.8/pages-for-subheaders/install-cluster-autoscaler' + to: "/v2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler", + from: "/v2.8/pages-for-subheaders/install-cluster-autoscaler", }, { - to: '/v2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster', - from: '/v2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster' + to: "/v2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster", + from: "/v2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster", }, { - to: '/v2.8/getting-started/installation-and-upgrade', - from: '/v2.8/pages-for-subheaders/installation-and-upgrade' + to: "/v2.8/getting-started/installation-and-upgrade", + from: "/v2.8/pages-for-subheaders/installation-and-upgrade", }, { - to: '/v2.8/getting-started/installation-and-upgrade/installation-references', - from: '/v2.8/pages-for-subheaders/installation-references' + to: "/v2.8/getting-started/installation-and-upgrade/installation-references", + from: "/v2.8/pages-for-subheaders/installation-references", }, { - to: '/v2.8/getting-started/installation-and-upgrade/installation-requirements', - from: '/v2.8/pages-for-subheaders/installation-requirements' + to: "/v2.8/getting-started/installation-and-upgrade/installation-requirements", + from: "/v2.8/pages-for-subheaders/installation-requirements", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/istio-setup-guide', - from: '/v2.8/pages-for-subheaders/istio-setup-guide' + to: "/v2.8/how-to-guides/advanced-user-guides/istio-setup-guide", + from: "/v2.8/pages-for-subheaders/istio-setup-guide", }, { - to: '/v2.8/integrations-in-rancher/istio/', - from: '/v2.8/pages-for-subheaders/istio' + to: "/v2.8/integrations-in-rancher/istio/", + from: "/v2.8/pages-for-subheaders/istio", }, { - to: '/v2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide', - from: '/v2.8/pages-for-subheaders/k3s-hardening-guide' + to: "/v2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide", + from: "/v2.8/pages-for-subheaders/k3s-hardening-guide", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup', - from: '/v2.8/pages-for-subheaders/kubernetes-cluster-setup' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup", + from: "/v2.8/pages-for-subheaders/kubernetes-cluster-setup", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup', - from: '/v2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup", + from: "/v2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup", }, { - to: '/v2.8/troubleshooting/kubernetes-components', - from: '/v2.8/pages-for-subheaders/kubernetes-components' + to: "/v2.8/troubleshooting/kubernetes-components", + from: "/v2.8/pages-for-subheaders/kubernetes-components", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/', - from: '/v2.8/pages-for-subheaders/kubernetes-resources-setup' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/", + from: "/v2.8/pages-for-subheaders/kubernetes-resources-setup", }, { - to: '/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher', - from: '/v2.8/pages-for-subheaders/launch-kubernetes-with-rancher' + to: "/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher", + from: "/v2.8/pages-for-subheaders/launch-kubernetes-with-rancher", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller', - from: '/v2.8/pages-for-subheaders/load-balancer-and-ingress-controller' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller", + from: "/v2.8/pages-for-subheaders/load-balancer-and-ingress-controller", }, { - to: '/v2.8/integrations-in-rancher/logging/', - from: '/v2.8/pages-for-subheaders/logging' + to: "/v2.8/integrations-in-rancher/logging/", + from: "/v2.8/pages-for-subheaders/logging", }, { - to: '/v2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration', - from: '/v2.8/pages-for-subheaders/machine-configuration' + to: "/v2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration", + from: "/v2.8/pages-for-subheaders/machine-configuration", }, { - to: '/v2.8/how-to-guides/new-user-guides/manage-clusters', - from: '/v2.8/pages-for-subheaders/manage-clusters' + to: "/v2.8/how-to-guides/new-user-guides/manage-clusters", + from: "/v2.8/pages-for-subheaders/manage-clusters", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas', - from: '/v2.8/pages-for-subheaders/manage-project-resource-quotas' + to: "/v2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas", + from: "/v2.8/pages-for-subheaders/manage-project-resource-quotas", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/manage-projects', - from: '/v2.8/pages-for-subheaders/manage-projects' + to: "/v2.8/how-to-guides/advanced-user-guides/manage-projects", + from: "/v2.8/pages-for-subheaders/manage-projects", }, { - to: '/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac', - from: '/v2.8/pages-for-subheaders/manage-role-based-access-control-rbac' + to: "/v2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac", + from: "/v2.8/pages-for-subheaders/manage-role-based-access-control-rbac", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides', - from: '/v2.8/pages-for-subheaders/monitoring-alerting-guides' + to: "/v2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides", + from: "/v2.8/pages-for-subheaders/monitoring-alerting-guides", }, { - to: '/v2.8/integrations-in-rancher/monitoring-and-alerting', - from: '/v2.8/pages-for-subheaders/monitoring-and-alerting' + to: "/v2.8/integrations-in-rancher/monitoring-and-alerting", + from: "/v2.8/pages-for-subheaders/monitoring-and-alerting", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides', - from: '/v2.8/pages-for-subheaders/monitoring-v2-configuration-guides' + to: "/v2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides", + from: "/v2.8/pages-for-subheaders/monitoring-v2-configuration-guides", }, { - to: '/v2.8/reference-guides/monitoring-v2-configuration', - from: '/v2.8/pages-for-subheaders/monitoring-v2-configuration' + to: "/v2.8/reference-guides/monitoring-v2-configuration", + from: "/v2.8/pages-for-subheaders/monitoring-v2-configuration", }, { - to: '/v2.8/how-to-guides/new-user-guides', - from: '/v2.8/pages-for-subheaders/new-user-guides' + to: "/v2.8/how-to-guides/new-user-guides", + from: "/v2.8/pages-for-subheaders/new-user-guides", }, { - to: '/v2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration', - from: '/v2.8/pages-for-subheaders/node-template-configuration' + to: "/v2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration", + from: "/v2.8/pages-for-subheaders/node-template-configuration", }, { - to: '/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix', - from: '/v2.8/pages-for-subheaders/nutanix' + to: "/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix", + from: "/v2.8/pages-for-subheaders/nutanix", }, { - to: '/v2.8/getting-started/installation-and-upgrade/other-installation-methods', - from: '/v2.8/pages-for-subheaders/other-installation-methods' + to: "/v2.8/getting-started/installation-and-upgrade/other-installation-methods", + from: "/v2.8/pages-for-subheaders/other-installation-methods", }, { - to: '/v2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides', - from: '/v2.8/pages-for-subheaders/prometheus-federator-guides' + to: "/v2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides", + from: "/v2.8/pages-for-subheaders/prometheus-federator-guides", }, { - to: '/v2.8/reference-guides/prometheus-federator', - from: '/v2.8/pages-for-subheaders/prometheus-federator' + to: "/v2.8/reference-guides/prometheus-federator", + from: "/v2.8/pages-for-subheaders/prometheus-federator", }, { - to: '/v2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples', - from: '/v2.8/pages-for-subheaders/provisioning-storage-examples' + to: "/v2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples", + from: "/v2.8/pages-for-subheaders/provisioning-storage-examples", }, { - to: '/v2.8/getting-started/quick-start-guides', - from: '/v2.8/pages-for-subheaders/quick-start-guides' + to: "/v2.8/getting-started/quick-start-guides", + from: "/v2.8/pages-for-subheaders/quick-start-guides", }, { - to: '/v2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy', - from: '/v2.8/pages-for-subheaders/rancher-behind-an-http-proxy' + to: "/v2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy", + from: "/v2.8/pages-for-subheaders/rancher-behind-an-http-proxy", }, { - to: '/v2.8/reference-guides/rancher-security/hardening-guides', - from: '/v2.8/pages-for-subheaders/rancher-hardening-guides' + to: "/v2.8/reference-guides/rancher-security/hardening-guides", + from: "/v2.8/pages-for-subheaders/rancher-hardening-guides", }, { - to: '/v2.8/reference-guides/best-practices/rancher-managed-clusters', - from: '/v2.8/pages-for-subheaders/rancher-managed-clusters' + to: "/v2.8/reference-guides/best-practices/rancher-managed-clusters", + from: "/v2.8/pages-for-subheaders/rancher-managed-clusters", }, { - to: '/v2.8/reference-guides/rancher-manager-architecture', - from: '/v2.8/pages-for-subheaders/rancher-manager-architecture' + to: "/v2.8/reference-guides/rancher-manager-architecture", + from: "/v2.8/pages-for-subheaders/rancher-manager-architecture", }, { - to: '/v2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker', - from: '/v2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker' + to: "/v2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker", + from: "/v2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker", }, { - to: '/v2.8/reference-guides/rancher-security', - from: '/v2.8/pages-for-subheaders/rancher-security' + to: "/v2.8/reference-guides/rancher-security", + from: "/v2.8/pages-for-subheaders/rancher-security", }, { - to: '/v2.8/reference-guides/cluster-configuration/rancher-server-configuration', - from: '/v2.8/pages-for-subheaders/rancher-server-configuration' + to: "/v2.8/reference-guides/cluster-configuration/rancher-server-configuration", + from: "/v2.8/pages-for-subheaders/rancher-server-configuration", }, { - to: '/v2.8/reference-guides/best-practices/rancher-server', - from: '/v2.8/pages-for-subheaders/rancher-server' + to: "/v2.8/reference-guides/best-practices/rancher-server", + from: "/v2.8/pages-for-subheaders/rancher-server", }, { - to: '/v2.8/getting-started/installation-and-upgrade/resources', - from: '/v2.8/pages-for-subheaders/resources' + to: "/v2.8/getting-started/installation-and-upgrade/resources", + from: "/v2.8/pages-for-subheaders/resources", }, { - to: '/v2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide', - from: '/v2.8/pages-for-subheaders/rke1-hardening-guide' + to: "/v2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide", + from: "/v2.8/pages-for-subheaders/rke1-hardening-guide", }, { - to: '/v2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide', - from: '/v2.8/pages-for-subheaders/rke2-hardening-guide' + to: "/v2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide", + from: "/v2.8/pages-for-subheaders/rke2-hardening-guide", }, { - to: '/v2.8/reference-guides/rancher-security/selinux-rpm', - from: '/v2.8/pages-for-subheaders/selinux-rpm' + to: "/v2.8/reference-guides/rancher-security/selinux-rpm", + from: "/v2.8/pages-for-subheaders/selinux-rpm", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers', - from: '/v2.8/pages-for-subheaders/set-up-cloud-providers' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers", + from: "/v2.8/pages-for-subheaders/set-up-cloud-providers", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers', - from: '/v2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers", + from: "/v2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers", }, { - to: '/v2.8/reference-guides/single-node-rancher-in-docker', - from: '/v2.8/pages-for-subheaders/single-node-rancher-in-docker' + to: "/v2.8/reference-guides/single-node-rancher-in-docker", + from: "/v2.8/pages-for-subheaders/single-node-rancher-in-docker", }, { - to: '/v2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes', - from: '/v2.8/pages-for-subheaders/use-existing-nodes' + to: "/v2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes", + from: "/v2.8/pages-for-subheaders/use-existing-nodes", }, { - to: '/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider', - from: '/v2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider' + to: "/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider", + from: "/v2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters', - from: '/v2.8/pages-for-subheaders/use-windows-clusters' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters", + from: "/v2.8/pages-for-subheaders/use-windows-clusters", }, { - to: '/v2.8/reference-guides/user-settings', - from: '/v2.8/pages-for-subheaders/user-settings' + to: "/v2.8/reference-guides/user-settings", + from: "/v2.8/pages-for-subheaders/user-settings", }, { - to: '/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere', - from: '/v2.8/pages-for-subheaders/vsphere' + to: "/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere", + from: "/v2.8/pages-for-subheaders/vsphere", }, { - to: '/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods', - from: '/v2.8/pages-for-subheaders/workloads-and-pods' + to: "/v2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods", + from: "/v2.8/pages-for-subheaders/workloads-and-pods", }, // Redirects for pages-for-subheaders removal [2.8] (end) - { // Redirects for pages-for-subheaders removal [latest] - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers', - from: '/pages-for-subheaders/about-provisioning-drivers' - }, - { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates', - from: '/pages-for-subheaders/about-rke1-templates' - }, - { - to: '/how-to-guides/new-user-guides/manage-clusters/access-clusters', - from: '/pages-for-subheaders/access-clusters' - }, - { - to: '/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration', - from: '/pages-for-subheaders/advanced-configuration' - }, - { - to: '/how-to-guides/advanced-user-guides', - from: '/pages-for-subheaders/advanced-user-guides' - }, - { - to: '/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install', - from: '/pages-for-subheaders/air-gapped-helm-cli-install' - }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config', - from: '/pages-for-subheaders/authentication-config' + // Redirects for pages-for-subheaders removal [latest] + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers", + from: "/pages-for-subheaders/about-provisioning-drivers", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration', - from: '/pages-for-subheaders/authentication-permissions-and-global-configuration' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates", + from: "/pages-for-subheaders/about-rke1-templates", }, { - to: '/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace', - from: '/pages-for-subheaders/aws-cloud-marketplace' + to: "/how-to-guides/new-user-guides/manage-clusters/access-clusters", + from: "/pages-for-subheaders/access-clusters", }, { - to: '/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery', - from: '/pages-for-subheaders/backup-restore-and-disaster-recovery' + to: "/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration", + from: "/pages-for-subheaders/advanced-configuration", }, { - to: '/reference-guides/backup-restore-configuration', - from: '/pages-for-subheaders/backup-restore-configuration' + to: "/how-to-guides/advanced-user-guides", + from: "/pages-for-subheaders/advanced-user-guides", }, { - to: '/reference-guides/best-practices', - from: '/pages-for-subheaders/best-practices' + to: "/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install", + from: "/pages-for-subheaders/air-gapped-helm-cli-install", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters', - from: '/pages-for-subheaders/checklist-for-production-ready-clusters' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config", + from: "/pages-for-subheaders/authentication-config", }, { - to: '/how-to-guides/advanced-user-guides/cis-scan-guides', - from: '/pages-for-subheaders/cis-scan-guides' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration", + from: "/pages-for-subheaders/authentication-permissions-and-global-configuration", }, { - to: '/integrations-in-rancher/cis-scans', - from: '/pages-for-subheaders/cis-scans' + to: "/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace", + from: "/pages-for-subheaders/aws-cloud-marketplace", }, { - to: '/reference-guides/cli-with-rancher', - from: '/pages-for-subheaders/cli-with-rancher' + to: "/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery", + from: "/pages-for-subheaders/backup-restore-and-disaster-recovery", }, { - to: '/integrations-in-rancher/cloud-marketplace', - from: '/pages-for-subheaders/cloud-marketplace' + to: "/reference-guides/backup-restore-configuration", + from: "/pages-for-subheaders/backup-restore-configuration", }, { - to: '/reference-guides/cluster-configuration', - from: '/pages-for-subheaders/cluster-configuration' + to: "/reference-guides/best-practices", + from: "/pages-for-subheaders/best-practices", }, { - to: '/integrations-in-rancher/istio/configuration-options', - from: '/pages-for-subheaders/configuration-options' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters", + from: "/pages-for-subheaders/checklist-for-production-ready-clusters", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml', - from: '/pages-for-subheaders/configure-microsoft-ad-federation-service-saml' + to: "/reference-guides/cli-with-rancher", + from: "/pages-for-subheaders/cli-with-rancher", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap', - from: '/pages-for-subheaders/configure-openldap' + to: "/integrations-in-rancher/cloud-marketplace", + from: "/pages-for-subheaders/cloud-marketplace", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml', - from: '/pages-for-subheaders/configure-shibboleth-saml' + to: "/reference-guides/cluster-configuration", + from: "/pages-for-subheaders/cluster-configuration", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage', - from: '/pages-for-subheaders/create-kubernetes-persistent-storage' + to: "/integrations-in-rancher/istio/configuration-options", + from: "/pages-for-subheaders/configuration-options", }, { - to: '/integrations-in-rancher/logging/custom-resource-configuration', - from: '/pages-for-subheaders/custom-resource-configuration' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml", + from: "/pages-for-subheaders/configure-microsoft-ad-federation-service-saml", }, { - to: '/getting-started/quick-start-guides/deploy-rancher-manager', - from: '/pages-for-subheaders/deploy-rancher-manager' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap", + from: "/pages-for-subheaders/configure-openldap", }, { - to: '/getting-started/quick-start-guides/deploy-workloads', - from: '/pages-for-subheaders/deploy-rancher-workloads' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml", + from: "/pages-for-subheaders/configure-shibboleth-saml", }, { - to: '/reference-guides/cluster-configuration/downstream-cluster-configuration', - from: '/pages-for-subheaders/downstream-cluster-configuration' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage", + from: "/pages-for-subheaders/create-kubernetes-persistent-storage", }, { - to: '/how-to-guides/advanced-user-guides/enable-experimental-features', - from: '/pages-for-subheaders/enable-experimental-features' + to: "/integrations-in-rancher/logging/custom-resource-configuration", + from: "/pages-for-subheaders/custom-resource-configuration", }, { - to: '/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration', - from: '/pages-for-subheaders/gke-cluster-configuration' + to: "/getting-started/quick-start-guides/deploy-rancher-manager", + from: "/pages-for-subheaders/deploy-rancher-manager", }, { - to: '/how-to-guides/new-user-guides/helm-charts-in-rancher', - from: '/pages-for-subheaders/helm-charts-in-rancher' + to: "/getting-started/quick-start-guides/deploy-workloads", + from: "/pages-for-subheaders/deploy-rancher-workloads", }, { - to: '/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler', - from: '/pages-for-subheaders/horizontal-pod-autoscaler' + to: "/reference-guides/cluster-configuration/downstream-cluster-configuration", + from: "/pages-for-subheaders/downstream-cluster-configuration", }, { - to: '/how-to-guides/new-user-guides/infrastructure-setup', - from: '/pages-for-subheaders/infrastructure-setup' + to: "/how-to-guides/advanced-user-guides/enable-experimental-features", + from: "/pages-for-subheaders/enable-experimental-features", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler', - from: '/pages-for-subheaders/install-cluster-autoscaler' + to: "/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration", + from: "/pages-for-subheaders/gke-cluster-configuration", }, { - to: '/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster', - from: '/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster' + to: "/how-to-guides/new-user-guides/helm-charts-in-rancher", + from: "/pages-for-subheaders/helm-charts-in-rancher", }, { - to: '/getting-started/installation-and-upgrade', - from: '/pages-for-subheaders/installation-and-upgrade' + to: "/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler", + from: "/pages-for-subheaders/horizontal-pod-autoscaler", }, { - to: '/getting-started/installation-and-upgrade/installation-references', - from: '/pages-for-subheaders/installation-references' + to: "/how-to-guides/new-user-guides/infrastructure-setup", + from: "/pages-for-subheaders/infrastructure-setup", }, { - to: '/getting-started/installation-and-upgrade/installation-requirements', - from: '/pages-for-subheaders/installation-requirements' + to: "/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler", + from: "/pages-for-subheaders/install-cluster-autoscaler", }, { - to: '/how-to-guides/advanced-user-guides/istio-setup-guide', - from: '/pages-for-subheaders/istio-setup-guide' + to: "/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster", + from: "/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster", }, { - to: '/integrations-in-rancher/istio', - from: '/pages-for-subheaders/istio' + to: "/getting-started/installation-and-upgrade", + from: "/pages-for-subheaders/installation-and-upgrade", }, { - to: '/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide', - from: '/pages-for-subheaders/k3s-hardening-guide' + to: "/getting-started/installation-and-upgrade/installation-references", + from: "/pages-for-subheaders/installation-references", }, { - to: '/how-to-guides/new-user-guides/kubernetes-cluster-setup', - from: '/pages-for-subheaders/kubernetes-cluster-setup' + to: "/getting-started/installation-and-upgrade/installation-requirements", + from: "/pages-for-subheaders/installation-requirements", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup', - from: '/pages-for-subheaders/kubernetes-clusters-in-rancher-setup' + to: "/how-to-guides/advanced-user-guides/istio-setup-guide", + from: "/pages-for-subheaders/istio-setup-guide", }, { - to: '/troubleshooting/kubernetes-components', - from: '/pages-for-subheaders/kubernetes-components' + to: "/integrations-in-rancher/istio", + from: "/pages-for-subheaders/istio", }, { - to: '/how-to-guides/new-user-guides/kubernetes-resources-setup', - from: '/pages-for-subheaders/kubernetes-resources-setup' + to: "/how-to-guides/new-user-guides/kubernetes-cluster-setup", + from: "/pages-for-subheaders/kubernetes-cluster-setup", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher', - from: '/pages-for-subheaders/launch-kubernetes-with-rancher' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup", + from: "/pages-for-subheaders/kubernetes-clusters-in-rancher-setup", }, { - to: '/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller', - from: '/pages-for-subheaders/load-balancer-and-ingress-controller' + to: "/troubleshooting/kubernetes-components", + from: "/pages-for-subheaders/kubernetes-components", }, { - to: '/integrations-in-rancher/logging', - from: '/pages-for-subheaders/logging' + to: "/how-to-guides/new-user-guides/kubernetes-resources-setup", + from: "/pages-for-subheaders/kubernetes-resources-setup", }, { - to: '/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration', - from: '/pages-for-subheaders/machine-configuration' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher", + from: "/pages-for-subheaders/launch-kubernetes-with-rancher", }, { - to: '/how-to-guides/new-user-guides/manage-clusters', - from: '/pages-for-subheaders/manage-clusters' + to: "/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller", + from: "/pages-for-subheaders/load-balancer-and-ingress-controller", }, { - to: '/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas', - from: '/pages-for-subheaders/manage-project-resource-quotas' + to: "/integrations-in-rancher/logging", + from: "/pages-for-subheaders/logging", }, { - to: '/how-to-guides/advanced-user-guides/manage-projects', - from: '/pages-for-subheaders/manage-projects' + to: "/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration", + from: "/pages-for-subheaders/machine-configuration", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac', - from: '/pages-for-subheaders/manage-role-based-access-control-rbac' + to: "/how-to-guides/new-user-guides/manage-clusters", + from: "/pages-for-subheaders/manage-clusters", }, { - to: '/how-to-guides/advanced-user-guides/monitoring-alerting-guides', - from: '/pages-for-subheaders/monitoring-alerting-guides' + to: "/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas", + from: "/pages-for-subheaders/manage-project-resource-quotas", }, { - to: '/integrations-in-rancher/monitoring-and-alerting', - from: '/pages-for-subheaders/monitoring-and-alerting' + to: "/how-to-guides/advanced-user-guides/manage-projects", + from: "/pages-for-subheaders/manage-projects", }, { - to: '/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides', - from: '/pages-for-subheaders/monitoring-v2-configuration-guides' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac", + from: "/pages-for-subheaders/manage-role-based-access-control-rbac", }, { - to: '/reference-guides/monitoring-v2-configuration', - from: '/pages-for-subheaders/monitoring-v2-configuration' + to: "/how-to-guides/advanced-user-guides/monitoring-alerting-guides", + from: "/pages-for-subheaders/monitoring-alerting-guides", }, { - to: '/how-to-guides/new-user-guides', - from: '/pages-for-subheaders/new-user-guides' + to: "/integrations-in-rancher/monitoring-and-alerting", + from: "/pages-for-subheaders/monitoring-and-alerting", }, { - to: '/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration', - from: '/pages-for-subheaders/node-template-configuration' + to: "/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides", + from: "/pages-for-subheaders/monitoring-v2-configuration-guides", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix', - from: '/pages-for-subheaders/nutanix' + to: "/reference-guides/monitoring-v2-configuration", + from: "/pages-for-subheaders/monitoring-v2-configuration", }, { - to: '/getting-started/installation-and-upgrade/other-installation-methods', - from: '/pages-for-subheaders/other-installation-methods' + to: "/how-to-guides/new-user-guides", + from: "/pages-for-subheaders/new-user-guides", }, { - to: '/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides', - from: '/pages-for-subheaders/prometheus-federator-guides' + to: "/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration", + from: "/pages-for-subheaders/node-template-configuration", }, { - to: '/reference-guides/prometheus-federator', - from: '/pages-for-subheaders/prometheus-federator' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix", + from: "/pages-for-subheaders/nutanix", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples', - from: '/pages-for-subheaders/provisioning-storage-examples' + to: "/getting-started/installation-and-upgrade/other-installation-methods", + from: "/pages-for-subheaders/other-installation-methods", }, { - to: '/getting-started/quick-start-guides', - from: '/pages-for-subheaders/quick-start-guides' + to: "/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides", + from: "/pages-for-subheaders/prometheus-federator-guides", }, { - to: '/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy', - from: '/pages-for-subheaders/rancher-behind-an-http-proxy' + to: "/reference-guides/prometheus-federator", + from: "/pages-for-subheaders/prometheus-federator", }, { - to: '/reference-guides/rancher-security/hardening-guides', - from: '/pages-for-subheaders/rancher-hardening-guides' + to: "/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples", + from: "/pages-for-subheaders/provisioning-storage-examples", }, { - to: '/reference-guides/best-practices/rancher-managed-clusters', - from: '/pages-for-subheaders/rancher-managed-clusters' + to: "/getting-started/quick-start-guides", + from: "/pages-for-subheaders/quick-start-guides", }, { - to: '/reference-guides/rancher-manager-architecture', - from: '/pages-for-subheaders/rancher-manager-architecture' + to: "/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy", + from: "/pages-for-subheaders/rancher-behind-an-http-proxy", }, { - to: '/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker', - from: '/pages-for-subheaders/rancher-on-a-single-node-with-docker' + to: "/reference-guides/best-practices/rancher-managed-clusters", + from: "/pages-for-subheaders/rancher-managed-clusters", }, { - to: '/reference-guides/rancher-security', - from: '/pages-for-subheaders/rancher-security' + to: "/reference-guides/rancher-manager-architecture", + from: "/pages-for-subheaders/rancher-manager-architecture", }, { - to: '/reference-guides/cluster-configuration/rancher-server-configuration', - from: '/pages-for-subheaders/rancher-server-configuration' + to: "/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker", + from: "/pages-for-subheaders/rancher-on-a-single-node-with-docker", }, { - to: '/reference-guides/best-practices/rancher-server', - from: '/pages-for-subheaders/rancher-server' + to: "/reference-guides/rancher-security", + from: "/pages-for-subheaders/rancher-security", }, { - to: '/getting-started/installation-and-upgrade/resources', - from: '/pages-for-subheaders/resources' + to: "/reference-guides/cluster-configuration/rancher-server-configuration", + from: "/pages-for-subheaders/rancher-server-configuration", }, { - to: '/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide', - from: '/pages-for-subheaders/rke1-hardening-guide' + to: "/reference-guides/best-practices/rancher-server", + from: "/pages-for-subheaders/rancher-server", }, { - to: '/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide', - from: '/pages-for-subheaders/rke2-hardening-guide' + to: "/getting-started/installation-and-upgrade/resources", + from: "/pages-for-subheaders/resources", }, { - to: '/reference-guides/rancher-security/selinux-rpm', - from: '/pages-for-subheaders/selinux-rpm' + to: "/reference-guides/rancher-security/selinux-rpm", + from: "/pages-for-subheaders/selinux-rpm", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers', - from: '/pages-for-subheaders/set-up-cloud-providers' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers", + from: "/pages-for-subheaders/set-up-cloud-providers", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers', - from: '/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers", + from: "/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers", }, { - to: '/reference-guides/single-node-rancher-in-docker', - from: '/pages-for-subheaders/single-node-rancher-in-docker' + to: "/reference-guides/single-node-rancher-in-docker", + from: "/pages-for-subheaders/single-node-rancher-in-docker", }, { - to: '/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes', - from: '/pages-for-subheaders/use-existing-nodes' + to: "/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes", + from: "/pages-for-subheaders/use-existing-nodes", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider', - from: '/pages-for-subheaders/use-new-nodes-in-an-infra-provider' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider", + from: "/pages-for-subheaders/use-new-nodes-in-an-infra-provider", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters', - from: '/pages-for-subheaders/use-windows-clusters' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters", + from: "/pages-for-subheaders/use-windows-clusters", }, { - to: '/reference-guides/user-settings', - from: '/pages-for-subheaders/user-settings' + to: "/reference-guides/user-settings", + from: "/pages-for-subheaders/user-settings", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere', - from: '/pages-for-subheaders/vsphere' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere", + from: "/pages-for-subheaders/vsphere", }, { - to: '/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods', - from: '/pages-for-subheaders/workloads-and-pods' + to: "/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods", + from: "/pages-for-subheaders/workloads-and-pods", }, // Redirects for pages-for-subheaders removal [latest] (end) - { // Redirects for dashboard v2.11 Preview (start) - to: '/v2.11/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth', - from: '/v2.11/admin-settings/authentication/google', + { + // Redirects for dashboard v2.11 Preview (start) + to: "/v2.11/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth", + from: "/v2.11/admin-settings/authentication/google", }, { - to: '/v2.11/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides', - from: '/v2.11/monitoring-alerting/configuration', + to: "/v2.11/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides", + from: "/v2.11/monitoring-alerting/configuration", }, { - to: '/v2.11/integrations-in-rancher/monitoring-and-alerting', - from: '/v2.11/monitoring-alerting', + to: "/v2.11/integrations-in-rancher/monitoring-and-alerting", + from: "/v2.11/monitoring-alerting", }, // Redirects for dashboard v2.11 Preview (end) - { // Redirects for dashboard#11114 (start) - to: '/v2.10/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth', - from: '/v2.10/admin-settings/authentication/google', + { + // Redirects for dashboard#11114 (start) + to: "/v2.10/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth", + from: "/v2.10/admin-settings/authentication/google", }, { - to: '/v2.10/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides', - from: '/v2.10/monitoring-alerting/configuration', + to: "/v2.10/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides", + from: "/v2.10/monitoring-alerting/configuration", }, { - to: '/v2.10/integrations-in-rancher/monitoring-and-alerting', - from: '/v2.10/monitoring-alerting', + to: "/v2.10/integrations-in-rancher/monitoring-and-alerting", + from: "/v2.10/monitoring-alerting", }, // Redirects for dashboard#11114 (end) - { // Redirects for dashboard#12040 (start) - to: '/v2.9/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth', - from: '/v2.9/admin-settings/authentication/google', + { + // Redirects for dashboard#12040 (start) + to: "/v2.9/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth", + from: "/v2.9/admin-settings/authentication/google", }, { - to: '/v2.9/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides', - from: '/v2.9/monitoring-alerting/configuration', + to: "/v2.9/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides", + from: "/v2.9/monitoring-alerting/configuration", }, { - to: '/v2.9/integrations-in-rancher/monitoring-and-alerting', - from: '/v2.9/monitoring-alerting', + to: "/v2.9/integrations-in-rancher/monitoring-and-alerting", + from: "/v2.9/monitoring-alerting", }, // Redirects for dashboard#12040 (end) - { // Redirects for dashboard#9970 - to: '/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences', - from: '/v2.8/cluster-provisioning/rke-clusters/behavior-differences-between-rke1-and-rke2/' + { + // Redirects for dashboard#9970 + to: "/v2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences", + from: "/v2.8/cluster-provisioning/rke-clusters/behavior-differences-between-rke1-and-rke2/", }, // Redirects for dashboard#9970 (end) - { // Redirects for restructure from PR #234 (start) - to: '/faq/general-faq', - from: '/faq' - }, { - to: '/troubleshooting/general-troubleshooting', - from: '/troubleshooting' + // Redirects for restructure from PR #234 (start) + to: "/faq/general-faq", + from: "/faq", }, { - to: '/getting-started/overview', - from: '/getting-started/introduction/overview' + to: "/troubleshooting/general-troubleshooting", + from: "/troubleshooting", }, { - to: '/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64', - from: '/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/rancher-on-arm64' + to: "/getting-started/overview", + from: "/getting-started/introduction/overview", }, { - to: '/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers', - from: '/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/unsupported-storage-drivers' + to: "/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64", + from: "/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/rancher-on-arm64", }, { - to: '/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features', - from: '/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/istio-traffic-management-features' + to: "/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers", + from: "/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/unsupported-storage-drivers", }, { - to: '/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery', - from: '/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/continuous-delivery' + to: "/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features", + from: "/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/istio-traffic-management-features", }, { - to: '/getting-started/installation-and-upgrade/installation-references/helm-chart-options', - from: '/reference-guides/installation-references/helm-chart-options' + to: "/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery", + from: "/getting-started/installation-and-upgrade/advanced-options/enable-experimental-features/continuous-delivery", }, { - to: '/getting-started/installation-and-upgrade/installation-references/tls-settings', - from: '/reference-guides/installation-references/tls-settings' + to: "/getting-started/installation-and-upgrade/installation-references/helm-chart-options", + from: "/reference-guides/installation-references/helm-chart-options", }, { - to: '/getting-started/installation-and-upgrade/installation-references/feature-flags', - from: '/reference-guides/installation-references/feature-flags' + to: "/getting-started/installation-and-upgrade/installation-references/tls-settings", + from: "/reference-guides/installation-references/tls-settings", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/manage-users-and-groups' + to: "/getting-started/installation-and-upgrade/installation-references/feature-flags", + from: "/reference-guides/installation-references/feature-flags", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/create-local-users' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/manage-users-and-groups", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-google-oauth' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/create-local-users", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-active-directory' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-google-oauth", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-freeipa' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-active-directory", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-azure-ad' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-freeipa", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-github' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-azure-ad", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-keycloak-oidc' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-github", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-keycloak-saml' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-keycloak-oidc", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-pingidentity' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-keycloak-saml", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-okta-saml' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-pingidentity", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/authentication-config/configure-okta-saml", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/configure-shibboleth-saml/about-group-permissions' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference', - from: '/reference-guides/configure-openldap/openldap-config-reference' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-authentication/configure-shibboleth-saml/about-group-permissions", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference", + from: "/reference-guides/configure-openldap/openldap-config-reference", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases", }, { - to: '/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding', - from: '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/custom-branding' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig', - from: '/how-to-guides/advanced-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig' + to: "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding", + from: "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/custom-branding", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint', - from: '/how-to-guides/advanced-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint' + to: "/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig", + from: "/how-to-guides/advanced-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters', - from: '/how-to-guides/advanced-user-guides/manage-clusters/access-clusters/add-users-to-clusters' + to: "/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint", + from: "/how-to-guides/advanced-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups', - from: '/how-to-guides/advanced-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups' + to: "/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters", + from: "/how-to-guides/advanced-user-guides/manage-clusters/access-clusters/add-users-to-clusters", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage' + to: "/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups", + from: "/how-to-guides/advanced-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/provisioning-storage-examples/persistent-storage-in-amazon-ebs' + to: "/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/provisioning-storage-examples/nfs-storage' + to: "/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/provisioning-storage-examples/persistent-storage-in-amazon-ebs", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage', - from: '/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/provisioning-storage-examples/vsphere-storage' + to: "/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/provisioning-storage-examples/nfs-storage", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces', - from: '/how-to-guides/advanced-user-guides/manage-clusters/projects-and-namespaces' + to: "/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage", + from: "/how-to-guides/advanced-user-guides/manage-clusters/create-kubernetes-persistent-storage/provisioning-storage-examples/vsphere-storage", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/rotate-certificates', - from: '/how-to-guides/advanced-user-guides/manage-clusters/rotate-certificates' + to: "/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces", + from: "/how-to-guides/advanced-user-guides/manage-clusters/projects-and-namespaces", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key', - from: '/how-to-guides/advanced-user-guides/manage-clusters/rotate-encryption-key' + to: "/how-to-guides/new-user-guides/manage-clusters/rotate-certificates", + from: "/how-to-guides/advanced-user-guides/manage-clusters/rotate-certificates", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates', - from: ['/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-cluster-templates', '/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-cluster-templates'] + to: "/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key", + from: "/how-to-guides/advanced-user-guides/manage-clusters/rotate-encryption-key", }, { - to: '/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools', - from: '/how-to-guides/advanced-user-guides/manage-clusters/nodes-and-node-pools' + to: "/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates", + from: [ + "/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-cluster-templates", + "/how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/manage-cluster-templates", + ], }, { - to: '/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes', - from: '/how-to-guides/advanced-user-guides/manage-clusters/clean-cluster-nodes' + to: "/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools", + from: "/how-to-guides/advanced-user-guides/manage-clusters/nodes-and-node-pools", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster' + to: "/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes", + from: "/how-to-guides/advanced-user-guides/manage-clusters/clean-cluster-nodes", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/rke1-vs-rke2-differences' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos", }, { - to: '/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/about-rancher-agents' + to: "/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/about-rancher-agents", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/azure-storageclass-configuration' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/azure-storageclass-configuration", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/windows-linux-cluster-feature-parity' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/windows-linux-cluster-feature-parity", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/network-requirements-for-host-gateway' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/network-requirements-for-host-gateway", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/workload-migration-guidance' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/use-windows-clusters/workload-migration-guidance", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/other-cloud-providers/amazon' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/other-cloud-providers/amazon", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-amazon', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-to-out-of-tree-amazon' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-amazon", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-to-out-of-tree-amazon", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/other-cloud-providers/azure' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/other-cloud-providers/azure", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/other-cloud-providers/google-compute-engine' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/other-cloud-providers/google-compute-engine", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/vsphere/configure-in-tree-vsphere' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/vsphere/configure-in-tree-vsphere", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/vsphere/configure-out-of-tree-vsphere' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/vsphere/configure-out-of-tree-vsphere", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/vsphere/migrate-from-in-tree-to-out-of-tree' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/set-up-cloud-providers/vsphere/migrate-from-in-tree-to-out-of-tree", }, { - to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-to-out-of-tree-vsphere', - }, - { to: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere', - from: '/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-to-out-of-tree-vsphere", }, { - to: '/how-to-guides/new-user-guides/add-users-to-projects', - from: '/how-to-guides/advanced-user-guides/manage-projects/add-users-to-projects' + to: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere", + from: "/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree", }, { - to: '/how-to-guides/new-user-guides/manage-namespaces', - from: '/how-to-guides/advanced-user-guides/manage-projects/manage-namespaces' + to: "/how-to-guides/new-user-guides/add-users-to-projects", + from: "/how-to-guides/advanced-user-guides/manage-projects/add-users-to-projects", }, { - to: '/how-to-guides/advanced-user-guides/open-ports-with-firewalld', - from: '/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/open-ports-with-firewalld' + to: "/how-to-guides/new-user-guides/manage-namespaces", + from: "/how-to-guides/advanced-user-guides/manage-projects/manage-namespaces", }, { - to: '/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs', - from: '/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/tune-etcd-for-large-installs' + to: "/how-to-guides/advanced-user-guides/open-ports-with-firewalld", + from: "/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/open-ports-with-firewalld", }, { - to: '/how-to-guides/advanced-user-guides/enable-api-audit-log', - from: '/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/enable-api-audit-log' + to: "/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs", + from: "/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/tune-etcd-for-large-installs", }, { - to: '/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer', - from: '/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/configure-layer-7-nginx-load-balancer' + to: "/how-to-guides/advanced-user-guides/enable-api-audit-log", + from: "/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/enable-api-audit-log", }, { - to: '/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements', - from: '/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements' + to: "/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer", + from: "/getting-started/installation-and-upgrade/advanced-options/advanced-use-cases/configure-layer-7-nginx-load-balancer", }, { - to: '/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter', - from: '/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter' + to: "/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements", + from: "/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements", }, { - to: '/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter', - from: '/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter' + to: "/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter", + from: "/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter", }, { - to: '/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues', - from: '/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues' + to: "/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter", + from: "/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter", }, { - to: '/integrations-in-rancher/cloud-marketplace/supportconfig', - from: '/explanations/integrations-in-rancher/cloud-marketplace/supportconfig' + to: "/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues", + from: "/explanations/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues", }, { - to: '/integrations-in-rancher/cis-scans/configuration-reference', - from: '/explanations/integrations-in-rancher/cis-scans/configuration-reference' + to: "/integrations-in-rancher/cloud-marketplace/supportconfig", + from: "/explanations/integrations-in-rancher/cloud-marketplace/supportconfig", }, { - to: '/integrations-in-rancher/cis-scans/rbac-for-cis-scans', - from: '/explanations/integrations-in-rancher/cis-scans/rbac-for-cis-scans' + to: "/integrations-in-rancher/fleet/architecture", + from: "/explanations/integrations-in-rancher/fleet-gitops-at-scale/architecture", }, { - to: '/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests', - from: '/explanations/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests' + to: "/integrations-in-rancher/fleet/windows-support", + from: "/explanations/integrations-in-rancher/fleet-gitops-at-scale/windows-support", }, { - to: '/integrations-in-rancher/cis-scans/custom-benchmark', - from: '/explanations/integrations-in-rancher/cis-scans/custom-benchmark' + to: "/integrations-in-rancher/fleet/use-fleet-behind-a-proxy", + from: "/explanations/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy", }, { - to: '/integrations-in-rancher/fleet/architecture', - from: '/explanations/integrations-in-rancher/fleet-gitops-at-scale/architecture' + to: "/integrations-in-rancher/harvester", + from: "/explanations/integrations-in-rancher/harvester", }, { - to: '/integrations-in-rancher/fleet/windows-support', - from: '/explanations/integrations-in-rancher/fleet-gitops-at-scale/windows-support' + to: "/integrations-in-rancher/istio/cpu-and-memory-allocations", + from: "/explanations/integrations-in-rancher/istio/cpu-and-memory-allocations", }, { - to: '/integrations-in-rancher/fleet/use-fleet-behind-a-proxy', - from: '/explanations/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy' + to: "/integrations-in-rancher/istio/rbac-for-istio", + from: "/explanations/integrations-in-rancher/istio/rbac-for-istio", }, { - to: '/integrations-in-rancher/harvester', - from: '/explanations/integrations-in-rancher/harvester' + to: "/integrations-in-rancher/istio/disable-istio", + from: "/explanations/integrations-in-rancher/istio/disable-istio", }, { - to: '/integrations-in-rancher/istio/cpu-and-memory-allocations', - from: '/explanations/integrations-in-rancher/istio/cpu-and-memory-allocations' + to: "/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations", + from: "/explanations/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations", }, { - to: '/integrations-in-rancher/istio/rbac-for-istio', - from: '/explanations/integrations-in-rancher/istio/rbac-for-istio' + to: "/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster", + from: "/explanations/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster", }, { - to: '/integrations-in-rancher/istio/disable-istio', - from: '/explanations/integrations-in-rancher/istio/disable-istio' + to: "/integrations-in-rancher/istio/configuration-options/project-network-isolation", + from: "/explanations/integrations-in-rancher/istio/configuration-options/project-network-isolation", }, { - to: '/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations', - from: '/explanations/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations' + to: "/integrations-in-rancher/longhorn", + from: "/explanations/integrations-in-rancher/longhorn", }, { - to: '/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster', - from: '/explanations/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster' + to: "/integrations-in-rancher/logging/logging-architecture", + from: "/explanations/integrations-in-rancher/logging/logging-architecture", }, { - to: '/integrations-in-rancher/istio/configuration-options/project-network-isolation', - from: '/explanations/integrations-in-rancher/istio/configuration-options/project-network-isolation' + to: "/integrations-in-rancher/logging/rbac-for-logging", + from: "/explanations/integrations-in-rancher/logging/rbac-for-logging", }, { - to: '/integrations-in-rancher/longhorn', - from: '/explanations/integrations-in-rancher/longhorn' + to: "/integrations-in-rancher/logging/logging-helm-chart-options", + from: "/explanations/integrations-in-rancher/logging/logging-helm-chart-options", }, { - to: '/integrations-in-rancher/logging/logging-architecture', - from: '/explanations/integrations-in-rancher/logging/logging-architecture' + to: "/integrations-in-rancher/logging/taints-and-tolerations", + from: "/explanations/integrations-in-rancher/logging/taints-and-tolerations", }, { - to: '/integrations-in-rancher/logging/rbac-for-logging', - from: '/explanations/integrations-in-rancher/logging/rbac-for-logging' + to: "/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows", + from: "/explanations/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows", }, { - to: '/integrations-in-rancher/logging/logging-helm-chart-options', - from: '/explanations/integrations-in-rancher/logging/logging-helm-chart-options' + to: "/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs", + from: "/explanations/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs", }, { - to: '/integrations-in-rancher/logging/taints-and-tolerations', - from: '/explanations/integrations-in-rancher/logging/taints-and-tolerations' + to: "/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works", + from: "/explanations/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works", }, { - to: '/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows', - from: '/explanations/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows' + to: "/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring", + from: "/explanations/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring", }, { - to: '/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs', - from: '/explanations/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs' + to: "/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards", + from: "/explanations/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards", }, { - to: '/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works', - from: '/explanations/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works' + to: "/integrations-in-rancher/monitoring-and-alerting/windows-support", + from: "/explanations/integrations-in-rancher/monitoring-and-alerting/windows-support", }, { - to: '/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring', - from: '/explanations/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring' + to: "/integrations-in-rancher/monitoring-and-alerting/promql-expressions", + from: "/explanations/integrations-in-rancher/monitoring-and-alerting/promql-expressions", }, { - to: '/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards', - from: '/explanations/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards' - }, - { - to: '/integrations-in-rancher/monitoring-and-alerting/windows-support', - from: '/explanations/integrations-in-rancher/monitoring-and-alerting/windows-support' - }, - { - to: '/integrations-in-rancher/monitoring-and-alerting/promql-expressions', - from: '/explanations/integrations-in-rancher/monitoring-and-alerting/promql-expressions' - }, - { - to: '/integrations-in-rancher/neuvector', - from: '/explanations/integrations-in-rancher/neuvector' - }, // Redirects for restructure from PR #234 (end) - { - to: '/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27', - from: '/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25' + to: "/integrations-in-rancher/neuvector", + from: "/explanations/integrations-in-rancher/neuvector", }, { - to: '/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale', - from: '/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher' + to: "/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale", + from: "/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher", }, // Redirects for restructure from PR #1147 (start) { - to: '/v2.8/api/v3-rancher-api-guide', - from: ['/v2.8/reference-guides/about-the-api', '/v2.8/pages-for-subheaders/about-the-api'] + to: "/v2.8/api/v3-rancher-api-guide", + from: [ + "/v2.8/reference-guides/about-the-api", + "/v2.8/pages-for-subheaders/about-the-api", + ], }, { - to: '/v2.8/api/api-tokens', - from: '/v2.8/reference-guides/about-the-api/api-tokens' + to: "/v2.8/api/api-tokens", + from: "/v2.8/reference-guides/about-the-api/api-tokens", }, { - to: '/api/v3-rancher-api-guide', - from: ['/reference-guides/about-the-api', '/pages-for-subheaders/about-the-api'] + to: "/api/v3-rancher-api-guide", + from: [ + "/reference-guides/about-the-api", + "/pages-for-subheaders/about-the-api", + ], }, { - to: '/api/api-tokens', - from: '/reference-guides/about-the-api/api-tokens' - } + to: "/api/api-tokens", + from: "/reference-guides/about-the-api/api-tokens", + }, // Redirects for restructure from PR #1147 (end) + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", + }, + { + from: "/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run", + to: "/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run", + }, + { + from: "/integrations-in-rancher/cis-scans/configuration-reference", + to: "/integrations-in-rancher/compliance-scans/configuration-reference", + }, + { + from: "/integrations-in-rancher/cis-scans/rbac-for-cis-scans", + to: "/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans", + }, + { + from: "/integrations-in-rancher/cis-scans/custom-benchmark", + to: "/integrations-in-rancher/compliance-scans/custom-benchmark", + }, ], }, ], ], scripts: [ { - src: 'https://cdn.cookielaw.org/scripttemplates/otSDKStub.js', - type:'text/javascript', - charset: 'UTF-8', - 'data-domain-script': '0f98beb0-fc4c-417d-a42e-564e2cae42d2', - async: true + src: "https://cdn.cookielaw.org/scripttemplates/otSDKStub.js", + type: "text/javascript", + charset: "UTF-8", + "data-domain-script": "0f98beb0-fc4c-417d-a42e-564e2cae42d2", + async: true, }, { - src: '/scripts/optanonwrapper.js', - type:'text/javascript', - async: true + src: "/scripts/optanonwrapper.js", + type: "text/javascript", + async: true, }, ], }; diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/api/api-reference.mdx b/i18n/zh/docusaurus-plugin-content-docs/current/api/api-reference.mdx index cb846d075bd..0ac99852235 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/api/api-reference.mdx +++ b/i18n/zh/docusaurus-plugin-content-docs/current/api/api-reference.mdx @@ -14,4 +14,4 @@ title: API 参考 import ApiDocMdx from '@theme/ApiDocMdx'; - + diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/api/v3-rancher-api-guide.md b/i18n/zh/docusaurus-plugin-content-docs/current/api/v3-rancher-api-guide.md index 89872d72845..a09d21eb6d7 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/api/v3-rancher-api-guide.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/api/v3-rancher-api-guide.md @@ -71,7 +71,7 @@ API 请求必须包含认证信息。认证是通过 [API 密钥](../reference-g ## 捕获 Rancher API 调用 -你可以使用浏览器开发人员工具来捕获 Rancher API 的调用方式。例如，你可以按照以下步骤使用 Chrome 开发人员工具来获取用于配置 RKE 集群的 API 调用： +You can use browser developer tools to capture how the v3 API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning a Rancher Kubernetes distribution cluster: 1. 在 Rancher UI 中，转到**集群管理**并单击**创建**。 1. 单击某个集群类型。此示例使用 Digital Ocean。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/contribute-to-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/current/contribute-to-rancher.md index baff7b735b9..ea433c1f2be 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/contribute-to-rancher.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/contribute-to-rancher.md @@ -35,7 +35,6 @@ title: 参与 Rancher 社区贡献 | (Rancher) Docker Machine | https://github.com/rancher/machine | 使用主机驱动时使用的 Docker Machine 二进制文件的源码仓库。这是 `docker/machine` 仓库的一个 fork。 | | machine-package | https://github.com/rancher/machine-package | 用于构建 Rancher Docker Machine 二进制文件。 | | kontainer-engine | https://github.com/rancher/kontainer-engine | kontainer-engine 的源码仓库，它是配置托管 Kubernetes 集群的工具。 | -| RKE repository | https://github.com/rancher/rke | Rancher Kubernetes Engine 的源码仓库，该工具可在任何主机上配置 Kubernetes 集群。 | | CLI | https://github.com/rancher/cli | Rancher 2.x 中使用的 Rancher CLI 的源码仓库。 | | (Rancher) Helm repository | https://github.com/rancher/helm | 打包的 Helm 二进制文件的源码仓库。这是 `helm/helm` 仓库的一个 fork。 | | Telemetry repository | https://github.com/rancher/telemetry | Telemetry 二进制文件的源码仓库。 | @@ -106,27 +105,6 @@ title: 参与 Rancher 社区贡献 -l app=rancher \ --timestamps=true ``` - - 在 RKE 集群的每个节点上使用 `docker` 的 Docker 安装 - - ``` - docker logs \ - --timestamps \ - $(docker ps | grep -E "rancher/rancher@|rancher_rancher" | awk '{ print $1 }') - ``` - - 使用 RKE 附加组件的 Kubernetes 安装 - - :::note - - 确保你配置了正确的 kubeconfig（例如，如果 Rancher Server 安装在 Kubernetes 集群上，则 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI 使用了嵌入式 kubectl。 - - ::: - - ``` - kubectl -n cattle-system \ - logs \ - --timestamps=true \ - -f $(kubectl --kubeconfig $KUBECONFIG get pods -n cattle-system -o json | jq -r '.items[] | select(.spec.containers[].name="cattle-server") | .metadata.name') - ``` - 系统日志记录（可能不存在，取决于操作系统） - `/var/log/messages` - `/var/log/syslog` diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/faq/container-network-interface-providers.md b/i18n/zh/docusaurus-plugin-content-docs/current/faq/container-network-interface-providers.md index 0d9c93e18fc..4a815e4a1f7 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/faq/container-network-interface-providers.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/faq/container-network-interface-providers.md @@ -43,58 +43,9 @@ CNI 网络插件使用封装网络模型（例如 Virtual Extensible Lan，缩 ## Rancher 提供哪些 CNI 插件？ -### RKE Kubernetes 集群 - -Rancher 开箱即用地为 RKE Kubernetes 集群提供了几个 CNI 网络插件，分别是 Canal、Flannel、Calico 和 Weave。 - -如果你使用 Rancher 创建新的 Kubernetes 集群，你可以选择你的 CNI 网络插件。 - -#### Canal - -![Canal Logo](/img/canal-logo.png) - -Canal 是一个 CNI 网络插件，它很好地结合了 Flannel 和 Calico 的优点。它让你轻松地将 Calico 和 Flannel 网络部署为统一的网络解决方案，将 Calico 的网络策略执行与 Calico（未封装）和 Flannel（封装）丰富的网络连接选项结合起来。 - -Canal 是 Rancher 默认的 CNI 网络插件，并采用了 Flannel 和 VXLAN 封装。 - -Kubernetes Worker 需要打开 UDP 端口 `8472` (VXLAN) 和 TCP 端口 `9099`（健康检查）。如果使用 Wireguard，则需要打开 UDP 端口 `51820` 和 `51821`。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 - -![](/img/canal-diagram.png) - -有关详细信息，请参阅 [Canal GitHub 页面](https://github.com/projectcalico/canal)。 - -#### Flannel - -![Flannel Logo](/img/flannel-logo.png) - -Flannel 是为 Kubernetes 配置 L3 网络结构的简单方法。Flannel 在每台主机上运行一个名为 flanneld 的二进制 Agent，该 Agent 负责从更大的预配置地址空间中为每台主机分配子网租约。Flannel 通过 Kubernetes API 或直接使用 etcd 来存储网络配置、分配的子网、以及其他辅助数据（例如主机的公共 IP）。数据包使用某种后端机制来转发，默认封装为 [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)。 - -默认情况下，封装的流量是不加密的。Flannel 提供了两种加密方案： - -* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec)：使用 [strongSwan](https://www.strongswan.org/) 在 Kubernetes worker 之间建立加密的 IPSec 隧道。它是加密的实验性后端。 -* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard)：比 strongSwan 更快的替代方案。 - -Kubernetes Worker 需要打开 UDP 端口 `8472` (VXLAN)。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络要求)。 - -![Flannel Diagram](/img/flannel-diagram.png) - -有关详细信息，请参阅 [Flannel GitHub 页面](https://github.com/flannel-io/flannel)。 - -#### Weave - -![Weave Logo](/img/weave-logo.png) - -Weave 在云上的 Kubernetes 集群中启用网络和网络策略。此外，它还支持加密对等节点之间的流量。 - -Kubernetes worker 需要打开 TCP 端口 `6783`（控制端口）、UDP 端口 `6783` 和 UDP 端口 `6784`（数据端口）。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络要求)。 - -有关详细信息，请参阅以下页面： - -- [Weave Net 官网](https://github.com/weaveworks/weave/blob/master/site/overview.md) - ### RKE2 Kubernetes 集群 -Rancher 开箱即用地为 RKE2 Kubernetes 集群提供了几个 CNI 网络插件，分别是 [Canal](#canal)（见上一节）、Calico 和 Cilium。 +Rancher 开箱即用地为 RKE2 Kubernetes 集群提供了几个 CNI 网络插件，分别是 Canal、Calico 和 Cilium。 如果你使用 Rancher 创建新的 Kubernetes 集群，你可以选择你的 CNI 网络插件。 @@ -125,6 +76,20 @@ Calico 还提供了一种无状态的 IP-in-IP 或 VXLAN 封装模式。如果 - [Project Calico 官方网站](https://www.projectcalico.org/) - [Calico 项目 GitHub 页面](https://github.com/projectcalico/calico) +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal 是一个 CNI 网络插件，它很好地结合了 Flannel 和 Calico 的优点。它让你轻松地将 Calico 和 Flannel 网络部署为统一的网络解决方案，将 Calico 的网络策略执行与 Calico（未封装）和 Flannel（封装）丰富的网络连接选项结合起来。 + +Canal 是 Rancher 默认的 CNI 网络插件，并采用了 Flannel 和 VXLAN 封装。 + +Kubernetes Worker 需要打开 UDP 端口 `8472` (VXLAN) 和 TCP 端口 `9099`（健康检查）。如果使用 Wireguard，则需要打开 UDP 端口 `51820` 和 `51821`。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +![](/img/canal-diagram.png) + +有关详细信息，请参阅 [Canal GitHub 页面](https://github.com/projectcalico/canal) + #### Cilium ![Cilium Logo](/img/cilium-logo.png) diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/faq/deprecated-features.md b/i18n/zh/docusaurus-plugin-content-docs/current/faq/deprecated-features.md index 18fd39792ad..01fd6892692 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/faq/deprecated-features.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/faq/deprecated-features.md @@ -16,10 +16,7 @@ Rancher 将在 GitHub 上发布的 Rancher 的[发版说明](https://github.com/ | Patch 版本 | 发布时间 | | ----------------------------------------------------------------- | ------------------ | -| [2.11.3](https://github.com/rancher/rancher/releases/tag/v2.11.2) | 2025 年 6 月 25 日 | -| [2.11.2](https://github.com/rancher/rancher/releases/tag/v2.11.2) | 2025 年 5 月 22 日 | -| [2.11.1](https://github.com/rancher/rancher/releases/tag/v2.11.1) | 2025 年 4 月 24 日 | -| [2.11.0](https://github.com/rancher/rancher/releases/tag/v2.11.0) | 2025 年 3 月 31 日 | +| [2.12.0](https://github.com/rancher/rancher/releases/tag/v2.12.0) | 2025 年 7 月 30 日 | ## 当一个功能被标记为弃用我可以得到什么样的预期？ diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/faq/dockershim.md b/i18n/zh/docusaurus-plugin-content-docs/current/faq/dockershim.md deleted file mode 100644 index cb1d658b131..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/current/faq/dockershim.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: Dockershim ---- - -Dockershim 是 Kubelet 和 Docker Daemon 之间的 CRI 兼容层。Kubernetes 1.20 版本宣布了[移除树内 Dockershim](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/)。目前计划在 Kubernetes 1.24 中移除。有关此移除的更多信息以及时间线，请参见 [Kubernetes Dockershim 弃用相关的常见问题](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 - -从 Kubernetes 1.21 开始。RKE 集群支持外部 Dockershim，来让用户继续使用 Docker 作为 CRI 运行时。现在，我们通过使用 [Mirantis 和 Docker ](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) 来确保 RKE 集群可以继续使用 Docker，从而实现上游开源社区的 Dockershim。 - -要启用外部 Dockershim，配置以下选项： - -``` -enable_cri_dockerd: true -``` - -如果你想使用其他容器运行时，Rancher 也提供使用 Containerd 作为默认运行时的，以边缘为中心的 K3s，和以数据中心为中心的 RKE2 Kubernetes 发行版。即使在 Kubernetes 1.24 删除了树内 Dockershim 之后，你也可以通过 Rancher 升级和管理导入的 RKE2 和 K3s Kubernetes 集群。 - -## 常见问题 - -
- -Q: 如果要获得 Rancher 对上游 Dockershim 的支持，我需要升级 Rancher 吗？ - -对于 RKE，Dockershim 的上游支持从 Kubernetes 1.21 开始。你需要使用 Rancher 2.6 或更高版本才能获取使用 Kubernetes 1.21 的 RKE 的支持。详情请参阅我们的[支持矩阵](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/)。 - -
- -Q: 我目前的 RKE 使用 Kubernetes 1.20。为了避免出现不再支持 Dockershim 的情况，我是否需要尽早将 RKE 升级到 Kubernetes 1.21？ - -A: 在使用 Kubernetes 1.20 的 RKE 中，Dockershim 版本依然可用，而且在 Kubernetes 1.24 之前不会在上游弃用。Kubernetes 会发出弃用 Dockershim 的警告，而 Rancher 在使用 Kubernetes 1.21 的 RKE 中已经缓解了这个问题。你可以按照计划正常升级到 Kubernetes 1.21，但也应该考虑在升级到 Kubernetes 1.22 时启用外部 Dockershim。在升级到 Kubernetes 1.24 之前，你需要启用外部 Dockershim，此时现有的实现都会被删除。 - -有关此移除的更多信息以及时间线，请参见 [Kubernetes Dockershim 弃用相关的常见问题](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 - -
- -Q: 如果我不想再依赖 Dockershim，我还有什么选择？ - -A: 你可以为 Kubernetes 使用不需要 Dockershim 支持的运行时，如 Containerd。RKE2 和 K3s 就是其中的两个选项。 - -
- -Q: 如果我目前使用 RKE1，但想切换到 RKE2，我可以怎样进行迁移？ - -A: Rancher 也在探索就地升级路径的可能性。此外，你始终可以使用 kubectl 将工作负载迁移到另一个集群。 - -
diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/faq/install-and-configure-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/current/faq/install-and-configure-kubectl.md index 2b9764b84f2..656fcba1ce7 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/faq/install-and-configure-kubectl.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/faq/install-and-configure-kubectl.md @@ -10,17 +10,11 @@ title: 安装和配置 kubectl ## 配置 -使用 RKE 创建 Kubernetes 集群时，RKE 会在本地目录中创建一个 `kube_config_cluster.yml`，该文件包含使用 `kubectl` 或 `helm` 等工具连接到新集群的凭证。 - -你可以将此文件复制为 `$HOME/.kube/config`。如果你使用多个 Kubernetes 集群，将 `KUBECONFIG` 环境变量设置为 `kube_config_cluster.yml` 的路径： - -``` -export KUBECONFIG=$(pwd)/kube_config_cluster.yml -``` +When you create a Kubernetes cluster with RKE2/K3s, the Kubeconfig file is stored at `/etc/rancher/rke2/rke2.yaml` or `/etc/rancher/k3s/k3s.yaml` depending on your chosen distribution. These files are used to configure access to the Kubernetes cluster. 使用 `kubectl` 测试你的连接性，并查看你是否可以获取节点列表： -``` +```shell kubectl get nodes NAME STATUS ROLES AGE VERSION 165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/faq/rancher-is-no-longer-needed.md b/i18n/zh/docusaurus-plugin-content-docs/current/faq/rancher-is-no-longer-needed.md index 0ae8e7be37b..48d4c445ffe 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/faq/rancher-is-no-longer-needed.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/faq/rancher-is-no-longer-needed.md @@ -12,9 +12,9 @@ title: 卸载 Rancher 如果删除了 Rancher，访问下游集群的方式取决于集群的类型和集群的创建方式。总而言之： -- **注册集群**：集群不受影响，你可以注册集群前的方法访问该集群。 +- **Registered/Imported clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. - **托管的 Kubernetes 集群**：如果你在 Kubernetes 云提供商（例如 EKS、GKE 或 AKS）中创建集群，你可以继续使用提供商的云凭证来管理集群。 -- **RKE 集群**：要访问 [RKE 集群](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md)，集群必须启用了[授权集群端点（authorized cluster endpoint，ACE）](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)，而且你必须从 Rancher UI 下载了集群的 kubeconfig 文件。RKE 集群默认启用授权集群端点。通过使用此端点，你可以直接使用 kubectl 访问你的集群，而不用通过 Rancher Server 的[认证代理](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-认证代理)进行通信。有关配置 kubectl 以使用授权集群端点的说明，请参阅[使用 kubectl 和 kubeconfig 文件直接访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验证)。这些集群将使用删除 Rancher 时配置的身份验证快照。 +- **Rancher provisioned clusters:** To access an [RKE2/K3s cluster](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. ## 如果我不想再使用 Rancher 了该怎么做？ @@ -51,7 +51,7 @@ title: 卸载 Rancher **结果**：注册的集群已与 Rancher 分离，并在 Rancher 外正常运行。 -## 如果我不想 Rancher 管理我的 RKE 集群或托管的 Kubernetes 集群该怎么办？ +## What if I don't want my hosted Kubernetes cluster managed by Rancher? 目前，我们没有将这些集群从 Rancher 中分离出来的功能。在这种情况下，“分离”指的是将 Rancher 组件移除出集群，并独立于 Rancher 管理对集群的访问。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/faq/technical-items.md b/i18n/zh/docusaurus-plugin-content-docs/current/faq/technical-items.md index b781a8cebb1..d9c83604fc8 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/faq/technical-items.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/faq/technical-items.md @@ -60,7 +60,7 @@ ClusterIP 是一个虚拟 IP，不会响应 ping。要测试 ClusterIP 是否配 ## Rancher 的状态存储在哪里？ - Docker 安装：在 `rancher/rancher` 容器的嵌入式 etcd 中，位于 `/var/lib/rancher`。 -- Kubernetes install：在为运行 Rancher 而创建的 RKE 集群的 etcd 中。 +- Kubernetes install: default location is in the `/var/lib/rancher/rke2` or `/var/lib/rancher/k3s` directories of the respective RKE2/K3s cluster created to run Rancher. ## 支持的 Docker 版本是如何确定的？ @@ -95,7 +95,7 @@ UI 由静态文件组成，并根据 API 的响应工作。换言之，UI 中可 ## 如何将其他参数/绑定/环境变量添加到 Rancher 启动的 Kubernetes 集群的 Kubernetes 组件中？ -你可以使用集群选项中的[配置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群配置文件参考)选项来添加其他参数/​​绑定/环境变量。有关详细信息，请参阅 RKE 文档中的[其他参数、绑定和环境变量](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/)，或浏览 [Cluster.ymls 示例](https://rancher.com/docs/rke/latest/en/example-yamls/)。 +You can add more arguments/binds/environment variables via the respective [RKE2 Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#cluster-configuration) or [K3s Config File](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md#cluster-configuration). ## 如何检查证书链是否有效？ diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md index cc4e205c941..15eb0a7f299 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md @@ -60,6 +60,12 @@ RKE2 基于使用上游[集群 API](https://github.com/kubernetes-sigs/cluster-a - 完全在 Rancher 中配置 RKE2 集群 - 除了 Canal 之外，还可以选择 CNI 选项， Calico、Cilium 和 Multus +When you make changes to your cluster configuration in RKE2, this may result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. + +The following are some specific example configuration changes that may cause the described behavior: + +- When editing the cluster and enabling drain before delete, the existing control plane nodes and worker are deleted and new nodes are created. + RKE2 配置还包括在具有 Windows 节点的集群上安装 RKE2。 RKE2 的 Windows 功能包括： diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md deleted file mode 100644 index 57c3c855491..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: RKE1 和 RKE2 差​​异 ---- - - - -RKE2，也称为 RKE Government，是一个完全符合标准的 Kubernetes 发行版，它专注于安全性和合规性。它被认为是 Rancher Kubernetes Engine（即 RKE1）的下一个迭代。 - -RKE1 和 RKE2 有一些细微的差异，本文将重点介绍这些差异。 - -## controlplane 组件 - -RKE1 使用 Docker 来部署和管理 controlplane 组件，还使用 Docker 作为 Kubernetes 的容器运行时。相比之下，RKE2 将 controlplane 组件作为由 kubelet 管理的静态 pod 启动。RKE2 的容器运行时是 Containerd，它允许 Mirror 容器镜像仓库等内容。使用 Docker 的 RKE1 不允许 Mirror。 - -## Cluster API - -RKE2/K3s 配置是基于 Cluster API (CAPI) 上游框架之上构建的，这导致 RKE2 配置的集群的行为通常与 RKE1 配置的集群不同。 - -如果你在 RKE2 中更改集群配置，节点**可能**会重新配置。这是由 CAPI 控制器而不是 Rancher 控制的。请注意，etcd 节点不适用相同的行为。 - -以下是一些可能导致上述行为的配置更改示例： - -- 编辑集群并启用`删除前清空`时，会删除现有的 controlplane 节点和 worker 并创建新节点。 - -如果你是习惯于 RKE1 配置的用户，请注意新的 RKE2 行为。 - -## 名词解释 - -从 RKE1 到 RKE2，某些术语已更改或已不再使用。例如，在 RKE1中，你使用**节点模板**，而在 RKE2 中，你可以在创建或编辑集群时配置集群节点池。另一个例子是 RKE1 中的**节点池（node pool）** 现在在 RKE2 中称为**主机池（machine pool）**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md index 4532ddf2e7b..327e035803b 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -44,14 +44,13 @@ Rancher 会发现并显示由 `kubectl` 创建的资源。但是在发现资源 ## 直接使用下游集群进行身份验证 -本节旨在帮助你设置访问 [RKE 集群的替代方法](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md)。 +本节旨在帮助你设置访问 [Rancher-launched 集群的替代方法](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md)。 -此方法仅适用于启用了[授权集群端点](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)的 RKE、RKE2 和 K3s集群。在 Rancher 创建集群时，Rancher 会生成一个 kubeconfig 文件，其中包含用于访问集群的额外 kubectl 上下文。该上下文允许你使用 kubectl 通过下游集群进行身份验证，而无需通过 Rancher 进行身份验证。有关授权集群端点如何工作的详细说明，请参阅[此页面](authorized-cluster-endpoint.md)。 +此方法仅适用于启用了[授权集群端点](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)的 RKE2 和 K3s集群。在 Rancher 创建集群时，Rancher 会生成一个 kubeconfig 文件，其中包含用于访问集群的额外 kubectl 上下文。该上下文允许你使用 kubectl 通过下游集群进行身份验证，而无需通过 Rancher 进行身份验证。有关授权集群端点如何工作的详细说明，请参阅[此页面](authorized-cluster-endpoint.md)。 在 RKE2 和 K3s 集群上，你需要[手动启用](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#对-rke2-和-k3s-集群的授权集群端点支持)授权集群端点。 -我们的最佳实践是使用此方法来访问 RKE、RKE2 和 K3s集群。这样，万一你无法连接到 Rancher，你仍然可以访问该集群。 - +我们的最佳实践是使用此方法来访问 RKE2 和 K3s集群。这样，万一你无法连接到 Rancher，你仍然可以访问该集群。 :::note 先决条件： @@ -70,7 +69,7 @@ CURRENT NAME CLUSTER AUTHINFO N 在此示例中，当你将 `kubectl` 与第一个上下文 `my-cluster` 一起使用时，你将通过 Rancher Server 进行身份验证。 -使用第二个上下文 `my-cluster-controlplane-1`，你将使用授权集群端点进行身份验证，直接与下游 RKE 集群通信。 +使用第二个上下文 `my-cluster-controlplane-1`，你将使用授权集群端点进行身份验证，直接与下游 RKE2/K3s 集群通信。 我们建议使用具有授权集群端点的负载均衡器。有关详细信息，请参阅[推荐的架构](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#授权集群端点架构)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md index 369a7be6f29..a2d3d658629 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -118,66 +118,7 @@ $ sudo shutdown -r now ## 清理节点 - - - -在运行以下命令之前，首先通过 Rancher UI 删除节点。 - -要删除节点： - -1. 点击 **☰** 并选择**集群管理**。 -1. 在集群表中，单击节点所属集群的名称。 -1. 在第一个选项卡中，单击节点状态旁边的复选框。 -1. 单击**删除**。 - -要删除整个集群而不是单个节点，或者不使用 Rancher UI 重新移动节点，请按照以下步骤操作： - -1. 从节点中[删除](#docker-容器镜像和卷) Docker 容器并[卸载](#挂载)任何卷。 -1. 重启节点。 -1. [删除](#目录和文件)所有剩余的文件。 -1. 确认在重新启动后已正确清理网络接口和 IP 表。否则请再重启一次。 - -### Windows 节点 - -要清理 Windows 节点，请运行 `c:\\etc\\rancher` 中的脚本。此脚本删除 Kubernetes 生成的资源和执行二进制文件，还会删除防火墙规则和网络设置： - -``` -pushd c:\etc\rancher -.\cleanup.ps1 -popd -``` - -运行此脚本后，节点将重置并可以重新添加到 Kubernetes 集群。 - -### Docker 容器、镜像和卷 - -:::caution - -清理 Docker 容器时要小心。以下命令将删除节点上的*所有* Docker 容器、镜像和卷，包括与 Rancher 无关的容器： - -::: - -``` -docker rm -f $(docker ps -qa) -docker rmi -f $(docker images -q) -docker volume rm $(docker volume ls -q) -``` - -### 挂载 - -Kubernetes 组件和 secret 会留下以下挂载： - -* `/var/lib/kubelet` -* `/var/lib/rancher` -* `/var/lib/kubelet/pods/` 中的其他挂载 - -要卸载所有挂载，请运行： - -``` -for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done -``` - - + :::note @@ -245,53 +186,6 @@ sudo k3s-uninstall.sh ::: - - -| 目录 | -|------------------------------| -| `/etc/ceph` | -| `/etc/cni` | -| `/etc/kubernetes` | -| `/opt/cni` | -| `/opt/rke` | -| `/run/calico` | -| `/run/flannel` | -| `/run/secrets/kubernetes.io` | -| `/var/lib/calico` | -| `/var/lib/cni` | -| `/var/lib/etcd` | -| `/var/lib/kubelet` | -| `/var/lib/rancher/rke` | -| `/var/lib/weave` | -| `/var/log/containers` | -| `/var/log/kube-audit` | -| `/var/log/pods` | -| `/var/run/calico` | - -**清理目录**： - -```shell -rm -rf /etc/ceph \ - /etc/cni \ - /etc/kubernetes \ - /opt/cni \ - /opt/rke \ - /run/calico \ - /run/flannel \ - /run/secrets/kubernetes.io \ - /var/lib/calico \ - /var/lib/cni \ - /var/lib/etcd \ - /var/lib/kubelet \ - /var/lib/rancher/rke \ - /var/lib/weave \ - /var/log/containers \ - /var/log/kube-audit \ - /var/log/pods \ - /var/run/calico -``` - - | 目录 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md index be1431337b7..e231727abbd 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -12,20 +12,6 @@ title: 证书轮换 可以为以下服务轮换证书： - - - -- etcd -- kubelet（节点证书） -- kubelet（服务证书，如果[启用](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)） -- kube-apiserver -- kube-proxy -- kube-scheduler -- kube-controller-manager - - - - - admin - api-server - controller-manager @@ -38,9 +24,6 @@ title: 证书轮换 - kubelet - kube-proxy - - - :::note 如果你未轮换 webhook 证书，且证书用了一年后已经过期，请参阅此[页面](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md)。 @@ -64,15 +47,4 @@ Rancher 启动的 Kubernetes 集群能够通过 UI 轮换自动生成的证书 ## 补充说明 - - - -虽然 RKE CLI 可以为 Kubernetes 集群组件使用自定义证书，但 Rancher 目前不允许在 Rancher 启动的 Kubernetes 集群中上传这些证书。 - - - - -在 RKE2 中，etcd 和 controlplane 节点都被视为相同的 `server`。因此，如果你轮换其中一个组件的服务证书，则两者的证书都会被轮换。证书只会针对指定的服务更改，但你会看到两个组件的节点都进入更新状态。你可能还会看到仅 Worker 节点进入更新状态。这是在证书更改后重启 Worker，以确保他们获得最新的客户端证书。 - - - +在 RKE2/K3s 中，etcd 和 controlplane 节点都被视为相同的 `server`。因此，如果你轮换其中一个组件的服务证书，则两者的证书都会被轮换。证书只会针对指定的服务更改，但你会看到两个组件的节点都进入更新状态。你可能还会看到仅 Worker 节点进入更新状态。这是在证书更改后重启 Worker，以确保他们获得最新的客户端证书。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md index b8c52cf70c5..f05c3b9d4d9 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -2,39 +2,11 @@ title: 加密密钥轮换 --- -## RKE1 加密密钥轮换 +:::note 重要 -1. 使用以下两个选项之一来启用加密密钥轮换： +加密密钥轮换默认启用，不能禁用。 - - 在 Rancher UI 中的 **Cluster Options > Advanced Options > Secrets Encryption** 下选择 `Enabled` 单选按钮： - - ![启用加密密钥轮换](/img/rke1-enable-secrets-encryption.png) - - - 或者，应用以下 YAML： - - ```yaml - rancher_kubernetes_engine_config: - services: - kube_api: - secrets_encryption_config: - enabled: true - ``` - -2. 在 Rancher UI 中轮换密钥： - - 2.1. 点击 **☰ > 集群管理**。 - - 2.2. 在所选集群旁边的屏幕最右侧选择 **⋮ > Rotate Encryption Keys**： - - ![加密密钥轮换](/img/rke1-encryption-key.png) - - - -## RKE2 加密密钥轮换 - -_**v2.6.7 新功能**_ - -> **重要提示**：加密密钥轮换默认启用，不能禁用。 +::: 要在 Rancher UI 中轮换密钥： @@ -44,5 +16,4 @@ _**v2.6.7 新功能**_ ![加密密钥轮换](/img/rke2-encryption-key.png) - -> **注意**：有关 RKE2 密文加密配置的更多信息，请参阅 [RKE2 文档](https://docs.rke2.io/security/secrets_encryption)。 \ No newline at end of file +> **注意**：有关 RKE2 密文加密配置的更多信息，请参阅 [RKE2 文档](https://docs.rke2.io/security/secrets_encryption)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md index c7f256a912f..d9258538bfc 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -15,10 +15,7 @@ title: 安装 Adapter | Rancher 版本 | Adapter 版本 | |-----------------|:----------------:| -| v2.11.3 | v106.0.0+up6.0.0 | -| v2.11.2 | v106.0.0+up6.0.0 | -| v2.11.1 | v106.0.0+up6.0.0 | -| v2.11.0 | v106.0.0+up6.0.0 | +| v2.12.0 | 107.0.0+up7.0.0 | ## 1. 获取对 Local 集群的访问权限 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/harvester/overview.md b/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/harvester/overview.md index b5ee0ff89eb..0453eeb700d 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/harvester/overview.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/integrations-in-rancher/harvester/overview.md @@ -24,7 +24,7 @@ Harvester 功能开关用于管理对 Rancher 中虚拟化管理（VM）页面 ## Harvester 主机驱动 -[Harvester 主机驱动](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/)通常可用于 Rancher 中的 RKE 和 RKE2 选项。无论 Harvester 功能开关是否启用，主机驱动都是可用的。请注意，主机驱动默认处于关闭状态。用户只能通过集群管理页面在 Harvester 上创建 RKE 或 RKE2 集群。 +[Harvester 主机驱动](https://docs.harvesterhci.io/v1.5/rancher/node/node-driver/)通常可用于 Rancher 中的 K3s 和 RKE2 选项。无论 Harvester 功能开关是否启用，主机驱动都是可用的。请注意，主机驱动默认处于关闭状态。用户只能通过集群管理页面在 Harvester 上创建 K3s 或 RKE2 集群。 Harvester 允许通过 Harvester UI 上传和显示 `.ISO` 镜像，但 Rancher UI 是不支持的。这是因为 `.ISO` 镜像通常需要额外的设置，这会干扰干净的部署（即无需用户干预），并且它们通常不用于云环境。 @@ -32,11 +32,11 @@ Harvester 允许通过 Harvester UI 上传和显示 `.ISO` 镜像，但 Rancher ## 端口要求 -Harvester 集群的端口要求可以在[此处](https://docs.harvesterhci.io/v1.1/install/requirements#networking)找到。 +Harvester 集群的端口要求可以在[此处](https://docs.harvesterhci.io/v1.5/install/requirements#networking)找到。 此外，其他网络注意事项如下： - 请务必为 VM VLAN 网络启用物理交换机的 VLAN 中继端口。 -- 按照[此处](https://docs.harvesterhci.io/v1.1/networking/index)的网络设置指南进行操作。 +- 按照[此处](https://docs.harvesterhci.io/v1.5/networking/index)的网络设置指南进行操作。 -对于其他集群（例如 K3s 和 RKE1）的其他端口要求，请参阅[这些文档](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters)。 +对于其他集群（例如 K3s 和 RKE2）的其他端口要求，请参阅[这些文档](https://docs.harvesterhci.io/v1.5/install/requirements/#guest-clusters)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md index c3c9b7a732d..e0b9eb6757f 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md @@ -16,4 +16,4 @@ While a managed cluster is disconnected from Rancher, management operations will - **Cleaning Up Disconnected Clusters**: Regularly remove clusters that will no longer reconnect to Rancher (e.g., clusters that have been decommissioned or destroyed). Keeping such clusters in the Rancher management system consumes unnecessary resources, which could impact Rancher's performance over time. -- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE/RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. +- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index afc1247854f..64d58392251 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -26,9 +26,7 @@ Rancher Logging 使用的是 [Logging Operator](https://github.com/kube-logging/ ### Kubernetes 组件 -_ClusterFlows_ 能够收集 Kubernetes 集群中所有主机上所有容器的日志。如果这些容器包含在 Kubernetes Pod 中，这个方法是适用的。但是，RKE 容器不存在于 Kubernetes 内。 - -目前，Rancher 能搜集 RKE 容器的日志，但不能轻易过滤。这是因为这些日志不包含源容器的信息（例如 `etcd` 或 `kube-apiserver`）。 +_ClusterFlows_ 能够收集 Kubernetes 集群中所有主机上所有容器的日志。如果这些容器包含在 Kubernetes Pod 中，这个方法是适用的。 Rancher 的未来版本将包含源容器名称，来支持过滤这些组件的日志。该功能实现之后，你将能够自定义 _ClusterFlow_ 来**仅**检索 Kubernetes 组件日志，并将日志发送到适当的输出位置。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index df057a54983..3e7a33d3e88 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -18,10 +18,6 @@ title: Rancher 运行技巧 在部署节点时，请遵循 K8s 和 etcd 的最佳实践，其中包括禁用 swap，检查集群中的所有主机之间是否有良好的网络连接，为每个节点使用唯一的主机名、MAC 地址和 `product_uuids`，检查所需端口是否已经打开，并使用配置 SSD 的 etcd 进行部署。详情请参见 [kubernetes 官方文档](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin)和 [etcd 性能操作指南](https://etcd.io/docs/v3.5/op-guide/performance/)。 -## 使用 RKE 时：备份状态文件（Statefile） - -RKE 将集群状态记录在一个名为 `cluster.rkestate` 的文件中，该文件对集群的恢复和/或通过 RKE 维护集群非常重要。由于这个文件包含证书材料，我们强烈建议在备份前对该文件进行加密。请在每次运行 `rke up` 后备份状态文件。 - ## 在同一个数据中心运行集群中的所有节点 为达到最佳性能，请在同一地理数据中心运行所有三个节点。如果你在云（如 AWS）上运行节点，请在不同的可用区（AZ）中运行这三个节点。例如，在 us-west-2a 中运行节点 1，在 us-west-2b 中运行节点 2，在 us-west-2c 中运行节点 3。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md index 154b01c5ad4..68ae57eae46 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md @@ -76,7 +76,7 @@ Rancher 使用两个 Kubernetes 应用程序资源：`apps.projects.cattle.io` ### 使用授权集群端点 (ACE) -[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点) (ACE) 提供了 Rancher 部署的 RKE、RKE2 和 K3s 集群的 Kubernetes API 访问。启用后，ACE 会为生成的 `kubeconfig` 文件配置直接访问下游集群 Endpoint，从而绕过 Rancher 代理。在可以直接访问下游集群 Kubernetes API 的场景下，可以减少 Rancher 负载。有关更多信息，请参阅[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)配置说明。 +[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点) (ACE) 提供了 Rancher 部署的 RKE2 和 K3s 集群的 Kubernetes API 访问。启用后，ACE 会为生成的 `kubeconfig` 文件配置直接访问下游集群 Endpoint，从而绕过 Rancher 代理。在可以直接访问下游集群 Kubernetes API 的场景下，可以减少 Rancher 负载。有关更多信息，请参阅[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)配置说明。 ### 减少 Event Handler 执行 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/cluster-configuration.md index 21f3b31f422..8add5f0520a 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/cluster-configuration.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/cluster-configuration.md @@ -14,7 +14,6 @@ title: 集群配置 集群配置选项取决于 Kubernetes 集群的类型： -- [RKE 集群配置](rancher-server-configuration/rke1-cluster-configuration.md) - [RKE2 集群配置](rancher-server-configuration/rke2-cluster-configuration.md) - [K3s 集群配置](rancher-server-configuration/k3s-cluster-configuration.md) - [EKS 集群配置](rancher-server-configuration/eks-cluster-configuration.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md index c3482cad893..684f10d8157 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md @@ -6,4 +6,4 @@ title: 下游集群配置 -以下文档将讨论[节点模板配置](node-template-configuration/node-template-configuration.md)和[主机配置](machine-configuration/machine-configuration.md)。 +以下文档将讨论[主机配置](machine-configuration/machine-configuration.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md index 4efbf1bb06b..9d42590068e 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md @@ -6,7 +6,6 @@ title: Rancher Server 配置 -- [RKE1 集群配置](rke1-cluster-configuration.md) - [RKE2 集群配置](rke2-cluster-configuration.md) - [K3s 集群配置](k3s-cluster-configuration.md) - [EKS 集群配置](eks-cluster-configuration.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/kubernetes-concepts.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/kubernetes-concepts.md index 5666ba5f4fc..6df41049db4 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/kubernetes-concepts.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/kubernetes-concepts.md @@ -6,13 +6,7 @@ title: Kubernetes 概念 ## 关于 Docker -Docker 是容器打包和运行时系统的标准。开发者在 Dockerfiles 中构建容器镜像，并在 Docker 镜像仓库中分发容器镜像。[Docker Hub](https://hub.docker.com) 是市面上主流的公有镜像仓库。许多企业还创建私有 Docker 镜像仓库。Docker 主要用于管理单个节点上的容器。 - -:::note - -由于 Kubernetes 已经成为了容器管理的主流工具，所以从 Rancher 2.x 版本开始，我们不再支持 Docker Swarm。如果你有使用 Docker 管理容器的需求，可以安装 Rancher 1.6 进行操作。 - -::: +Docker 是一个容器打包和运行时标准。开发者可以使用 Dockerfile 构建容器镜像，并从 Docker 镜像仓库分发容器镜像。[Docker Hub](https://hub.docker.com) 是一个流行的公共镜像仓库。许多组织也建立了私有 Docker 镜像仓库。Docker 主要用于管理单个节点上的容器。 ## 关于 Kubernetes @@ -22,7 +16,7 @@ Kubernetes 是容器和集群管理的标准。YAML 文件规定了组成一个 集群是可作为一个系统协同工作的一组计算机。 -_Kubernetes 集群_ 是使用 [Kubernetes 容器编排系统](https://kubernetes.io/)来部署、运维和伸缩 Docker 容器的集群，让你的组织实现应用自动化运维。 +_Kubernetes 集群_ 是使用 [Kubernetes 容器编排系统](https://kubernetes.io/)来部署、运维和伸缩 容器的集群，让你的组织实现应用自动化运维。 ## Kubernetes 集群中节点的角色 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/architecture-recommendations.md index 7e61817fbe6..2b518f95407 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -28,14 +28,6 @@ title: 架构推荐 ![运行 Rancher Management Server 的 K3s Kubernetes 集群的架构](/img/k3s-server-storage.svg) -### RKE Kubernetes 集群安装 - -在 RKE 安装中，集群数据在集群中的三个 etcd 节点上复制，以在某个节点发生故障时提供冗余和进行数据复制。 - -
运行 Rancher Management Server 的 RKE Kubernetes 集群的架构
- -![运行 Rancher Management Server 的 RKE Kubernetes 集群的架构](/img/rke-server-storage.svg) - ## Kubernetes 安装的负载均衡器推荐配置 我们建议你为负载均衡器和 Ingress Controller 使用以下配置： @@ -57,7 +49,7 @@ title: 架构推荐 ## Kubernetes 安装的推荐节点角色 -如果 Rancher 安装在 K3s Kubernetes 或 RKE Kubernetes 集群上，以下建议适用。 +如果 Rancher 安装在 K3s Kubernetes 上，则适用以下建议。 ### K3s 集群角色 @@ -65,38 +57,6 @@ title: 架构推荐 对于运行 Rancher Management Server 的集群，我们建议使用两个 server 节点。不需要 Agent 节点。 -### RKE 集群角色 - -如果 Rancher 安装在 RKE Kubernetes 集群上，该集群应具有三个节点，并且每个节点都应具有所有三个 Kubernetes 角色，分别是 etcd，controlplane 和 worker。 - -### Rancher Server 和下游 Kubernetes 集群的 RKE 集群架构对比 - -我们对 Rancher Server 集群上 RKE 节点角色建议，与对运行你的应用和服务的下游集群的建议相反。 - -在配置下游 Kubernetes 集群时，Rancher 使用 RKE 作为创建下游 Kubernetes 集群的工具。注意：Rancher 将在未来的版本中添加配置下游 K3s 集群的功能。 - -我们建议下游 Kubernetes 集群中的每个节点都只分配一个角色，以确保稳定性和可扩展性。 - -![Rancher Server 集群中和下游集群中节点的 Kubernetes 角色对比](/img/rancher-architecture-node-roles.svg) - -RKE 每个角色至少需要一个节点，但并不强制每个节点只能有一个角色。但是，我们建议为运行应用的集群中的每个节点，使用单独的角色，以保证在服务拓展时，worker 节点上的工作负载不影响 Kubernetes master 或集群的数据。 - -以下是我们对下游集群的最低配置建议： - -- **三个仅使用 etcd 角色的节点** ，以在三个节点中其中一个发生故障时，仍能保障集群的高可用性。 -- **两个只有 controlplane 角色的节点** ，以保证 master 组件的高可用性。 -- **一个或多个只有 worker 角色的节点**，用于运行 Kubernetes 节点组件，以及你部署的服务或应用的工作负载。 - -在设置 Rancher Server 时，在三个节点上使用全部这三个角色也是安全的，因为： - -* 它允许一个 `etcd` 节点故障。 -* 它通过多个 `controlplane` 节点来维护 master 组件的多个实例。 -* 此集群上没有创建除 Rancher 之外的其他工作负载。 - -由于 Rancher Server 集群中没有部署其他工作负载，因此在大多数情况下，这个集群都不需要使用我们出于可扩展性和可用性的考虑，而为下游集群推荐的架构。 - -有关下游集群的最佳实践，请查看[生产环境清单](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/checklist-for-production-ready-clusters.md)或[最佳实践](../best-practices/best-practices.md)。 - ## 授权集群端点架构 如果你使用[授权集群端点（ACE）](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)，我们建议你创建一个指向负载均衡器的 FQDN，这个负载均衡器把流量转到所有角色为 `controlplane` 的节点。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index 9d7570ba151..21326c85789 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -39,7 +39,7 @@ Rancher 使用 [ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-c - 检测下游集群中的资源变化 - 将下游集群的当前状态变更到目标状态 - 配置集群和项目的访问控制策略 -- 通过调用所需的 Docker Machine 驱动和 Kubernetes 引擎（例如 RKE 和 GKE）来配置集群 +- 通过调用所需的 Docker Machine 驱动和 Kubernetes 引擎（例如，GKE）来配置集群 默认情况下，Cluster Controller 连接到 Cluster Agent，Rancher 才能与下游集群通信。如果 Cluster Agent 不可用，Cluster Controller 可以连接到 [Node Agent](#3-node-agents)。 @@ -60,7 +60,7 @@ Cluster Agent，也叫做 `cattle-cluster-agent`，是运行在下游集群中 授权集群端点（ACE）可连接到下游集群的 Kubernetes API Server，而不用通过 Rancher 认证代理调度请求。 -> 授权集群端点仅适用于 Rancher 启动的 Kubernetes 集群，即只适用于 Rancher [使用 RKE](../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) 来配置的集群。它不适用于导入的集群，也不适用于托管在 Kubernetes 提供商中的集群（例如 Amazon 的 EKS）。 +> 授权集群端点仅适用于 Rancher 启动的 Kubernetes 集群，即 [Rancher 配置的集群](../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) 它不适用于导入的集群，也不适用于托管在 Kubernetes 提供商中的集群（例如 Amazon 的 EKS）。 授权集群端点的主要用途： @@ -81,11 +81,8 @@ Cluster Agent，也叫做 `cattle-cluster-agent`，是运行在下游集群中 维护、排除问题和升级集群需要用到以下文件，请妥善保管这些文件： -- `rancher-cluster.yml`：RKE 集群配置文件。 -- `kube_config_rancher-cluster.yml`：集群的 Kubeconfig 文件，包含完全访问集群的凭证。如果 Rancher 出现问题时，你可以使用此文件认证由 Rancher 启动的 Kubernetes 集群。 -- `rancher-cluster.rkestate`：Kubernetes 集群状态文件，文件包含用于完全访问集群的凭证。注意：仅在使用 RKE v0.2.0 或更高版本时，才会创建此该文件。 - -> **注意**：后两个文件名中的 `rancher-cluster` 部分取决于你命名 RKE 集群配置文件的方式。 +- `config.yaml`: The RKE2 and K3s cluster configuration file. +- `rke2.yaml` or `k3s.yaml`: The Kubeconfig file for your RKE2 or K3s cluster. This file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. 有关在没有 Rancher 认证代理和其他配置选项的情况下连接到集群的更多信息，请参见 [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 @@ -97,13 +94,7 @@ Rancher 使用什么工具配置下游集群，取决于集群的类型。 Rancher 可以动态启动云上（如 Amazon EC2、DigitalOcean、Azure 或 vSphere 等）的节点，然后在节点上安装 Kubernetes。 -Rancher 使用 [RKE](https://github.com/rancher/rke) 和 [docker-machine](https://github.com/rancher/machine) 来配置这类型的集群。 - -### Rancher 为自定义节点启动 Kubernetes - -在配置此类集群时，Rancher 会在现有节点上安装 Kubernetes，从而创建自定义集群。 - -Rancher 使用 [RKE](https://github.com/rancher/rke) 来启动此类集群。 +Rancher 使用 [docker-machine](https://github.com/rancher/machine) 来配置这类型的集群。 ### 托管的 Kubernetes 提供商 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/hardening-guides.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/hardening-guides.md index 8dc764b2f9a..799268cdd1e 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/hardening-guides.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/hardening-guides.md @@ -12,7 +12,6 @@ Rancher 为每个受支持的 Rancher 版本的 Kubernetes 发行版提供了特 Rancher 使用以下 Kubernetes 发行版： -- [**RKE**](https://rancher.com/docs/rke/latest/en/)（Rancher Kubernetes Engine）是经过 CNCF 认证的 Kubernetes 发行版，完全在 Docker 容器中运行。 - [**RKE2**](https://docs.rke2.io/) 是一个完全合规的 Kubernetes 发行版，专注于安全和合规性。 - [**K3s**](https://docs.k3s.io/) 是一个完全合规的，轻量级 Kubernetes 发行版。它易于安装，内存需求只有上游 Kubernetes 的一半，所有组件都在一个小于 100 MB 的二进制文件中。 @@ -22,12 +21,6 @@ Rancher 使用以下 Kubernetes 发行版： 每个自我评估指南都附有强化指南。这些指南与列出的 Rancher 版本一起进行了测试。每个自我评估指南都在特定的 Kubernetes 版本和 CIS Benchmark 版本上进行了测试。如果 CIS Benchmark 尚未针对你的 Kubernetes 版本进行验证，你可以使用现有指南，直到添加适合你的版本的指南。 -### RKE 指南 - -| Kubernetes 版本 | CIS Benchmark 版本 | 自我评估指南 | 加固指南 | -|--------------------|-----------------------|-----------------------|------------------| -| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [链接](rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [链接](rke1-hardening-guide/rke1-hardening-guide.md) | - ### RKE2 指南 | 类型 | Kubernetes 版本 | CIS Benchmark 版本 | 自我评估指南 | 加固指南 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md deleted file mode 100644 index eaffdb72d92..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md +++ /dev/null @@ -1,516 +0,0 @@ ---- -title: RKE 加固指南 ---- - - - - - - - -本文档提供了针对生产环境的 RKE 集群进行加固的具体指导，以便在使用 Rancher 部署之前进行配置。它概述了满足信息安全中心（Center for Information Security, CIS）Kubernetes benchmark controls 所需的配置和控制。 - -:::note -这份加固指南描述了如何确保你集群中的节点安全。我们建议你在安装 Kubernetes 之前遵循本指南。 -::: - -此加固指南适用于 RKE 集群，并与以下版本的 CIS Kubernetes Benchmark、Kubernetes 和 Rancher 相关联： - -| Rancher 版本 | CIS Benchmark 版本 | Kubernetes 版本 | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 至 v1.26 | - -:::note -- 在 Benchmark v1.24 及更高版本中，检查 id `4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated)` 可能会失败，因为 `/etc/kubernetes/ssl/kube-ca.pem` 默认设置为 644。 -- 在 Benchmark v1.7 中，不再需要 `--protect-kernel-defaults` (`4.2.6`) 参数，并已被 CIS 删除。 -::: - -有关如何评估加固的 RKE 集群与官方 CIS benchmark 的更多细节，请参考特定 Kubernetes 和 CIS benchmark 版本的 RKE 自我评估指南。 - -## 主机级别要求 - -### 配置 Kernel 运行时参数 - -建议对群集中的所有节点类型使用以下 `sysctl` 配置。在 `/etc/sysctl.d/90-kubelet.conf` 中设置以下参数： - -```ini -vm.overcommit_memory=1 -vm.panic_on_oom=0 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -运行 `sysctl -p /etc/sysctl.d/90-kubelet.conf` 以启用设置。 - -### 配置 `etcd` 用户和组 - -在安装 RKE 之前，需要设置 **etcd** 服务的用户帐户和组。 - -#### 创建 `etcd` 用户和组 - -要创建 **etcd** 用户和组，请运行以下控制台命令。 -下面的命令示例中使用 `52034` 作为 **uid** 和 **gid** 。 -任何有效且未使用的 **uid** 或 **gid** 都可以代替 `52034`。 - -```bash -groupadd --gid 52034 etcd -useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin -``` - -在通过集群配置文件 `config.yml` 部署RKE时，请更新 `etcd` 用户的 `uid` 和 `gid`： - -```yaml -services: - etcd: - gid: 52034 - uid: 52034 -``` - -## Kubernetes 运行时要求 - -### 配置 `default` Service Account - -#### 设置 `automountServiceAccountToken` 为 `false` 用于 `default` service accounts - -Kubernetes 提供了一个 default service account，供集群工作负载使用，其中没有为 pod 分配特定的 service account。 -如果需要从 pod 访问 Kubernetes API，则应为该 pod 创建特定的 service account，并向该 service account 授予权限。 -应配置 default service account，使其不提供 service account 令牌，并且不应具有任何明确的权限分配。 - -对于标准 RKE 安装上的每个命名空间（包括 `default` 和 `kube-system`），`default` service account 必须包含以下值： - -```yaml -automountServiceAccountToken: false -``` - -将以下配置保存到名为 `account_update.yaml` 的文件中。 - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -创建一个名为 `account_update.yaml` 的 bash 脚本文件。 -确保执行 `chmod +x account_update.sh` 命令，以赋予脚本执行权限。 - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -执行此脚本将 `account_update.yaml` 配置应用到所有命名空间中的 `default` service account。 - -### 配置网络策略 - -#### 确保所有命名空间都定义了网络策略 - -在同一个 Kubernetes 集群上运行不同的应用程序会带来风险，即某个受感染的应用程序可能会攻击相邻的应用程序。为确保容器只与其预期通信的容器进行通信，网络分段至关重要。网络策略规定了哪些 Pod 可以互相通信，以及与其他网络终端通信的方式。 - -网络策略是命名空间范围的。当在特定命名空间引入网络策略时，所有未被策略允许的流量将被拒绝。然而，如果在命名空间中没有网络策略，那么所有流量将被允许进入和离开该命名空间中的 Pod。要强制执行网络策略，必须启用容器网络接口（container network interface, CNI）插件。本指南使用 [Canal](https://github.com/projectcalico/canal) 来提供策略执行。有关 CNI 提供程序的其他信息可以在[这里](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/)找到。 - -一旦在集群上启用了 CNI 提供程序，就可以应用默认的网络策略。下面提供了一个 **permissive** 的示例供参考。如果你希望允许匹配某个命名空间中所有 Pod 的所有入站和出站流量（即使添加了策略导致某些 Pod 被视为”隔离”），你可以创建一个明确允许该命名空间中所有流量的策略。请将以下配置保存为 `default-allow-all.yaml`。有关网络策略的其他[文档](https://kubernetes.io/docs/concepts/services-networking/network-policies/)可以在 Kubernetes 站点上找到。 - -:::caution -此网络策略只是一个示例，不建议用于生产用途。 -::: - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-allow-all -spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress -``` - -创建一个名为 `apply_networkPolicy_to_all_ns.sh`的 Bash 脚本文件。 - -确保运行 `chmod +x apply_networkPolicy_to_all_ns.sh` 命令，以赋予脚本执行权限。 - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl apply -f default-allow-all.yaml -n ${namespace} -done -``` - -执行此脚本以将 `default-allow-all.yaml` 配置和 **permissive** 的 `NetworkPolicy` 应用于所有命名空间。 - -## 已知限制 - -- 当注册自定义节点仅提供公共 IP 时，Rancher **exec shell** 和 **查看 pod 日志** 在加固设置中**不起作用**。 此功能需要在注册自定义节点时提供私有 IP。 - -## 加固的 RKE `cluster.yml` 配置参考 - -参考的 `cluster.yml` 文件是由 RKE CLI 使用的，它提供了实现 RKE 加固安装所需的配置。 -RKE [文档](https://rancher.com/docs/rke/latest/en/installation/)提供了有关配置项的更多详细信息。这里参考的 `cluster.yml` 不包括必需的 `nodes` 指令，因为它取决于你的环境。在 RKE 中有关节点配置的文档可以在[这里](https://rancher.com/docs/rke/latest/en/config-options/nodes/)找到。 - -示例 `cluster.yml` 配置文件中包含了一个 Admission Configuration 策略，在 `services.kube-api.admission_configuration` 字段中指定。这个[示例](../../psa-restricted-exemptions.md)策略包含了命名空间的豁免规则，这对于在Rancher中正确运行导入的RKE集群非常必要，类似于Rancher预定义的 [`rancher-restricted`](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) 策略。 - -如果你希望使用 RKE 的默认 `restricted` 策略，则将 `services.kube-api.admission_configuration` 字段留空，并将 `services.pod_security_configuration` 设置为 `restricted`。你可以在 [RKE 文档](https://rke.docs.rancher.com/config-options/services/pod-security-admission)中找到更多信息。 - - - - -:::note -如果你打算将一个 RKE 集群导入到 Rancher 中，请参考此[文档](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md)以了解如何配置 PSA 以豁免 Rancher 系统命名空间。 -::: - -```yaml -# 如果你打算在离线环境部署 Kubernetes， -# 请查阅文档以了解如何配置自定义的 RKE 镜像。 -nodes: [] -kubernetes_version: # 定义 RKE 版本 -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - # 如果你在 `admission_configuration` 中设置了自定义策略， - # 请将 `pod_security_configuration` 字段留空。 - # 否则，将其设置为 `restricted` 以使用 RKE 预定义的受限策略， - # 并删除 `admission_configuration` 字段中的所有内容。 - # - # pod_security_configuration: restricted - # - admission_configuration: - apiVersion: apiserver.config.k8s.io/v1 - kind: AdmissionConfiguration - plugins: - - name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1 - kind: PodSecurityConfiguration - defaults: - enforce: "restricted" - enforce-version: "latest" - audit: "restricted" - audit-version: "latest" - warn: "restricted" - warn-version: "latest" - exemptions: - usernames: [] - runtimeClasses: [] - namespaces: [calico-apiserver, - calico-system, - cattle-alerting, - cattle-csp-adapter-system, - cattle-elemental-system, - cattle-epinio-system, - cattle-externalip-system, - cattle-fleet-local-system, - cattle-fleet-system, - cattle-gatekeeper-system, - cattle-global-data, - cattle-global-nt, - cattle-impersonation-system, - cattle-istio, - cattle-istio-system, - cattle-logging, - cattle-logging-system, - cattle-monitoring-system, - cattle-neuvector-system, - cattle-prometheus, - cattle-provisioning-capi-system, - cattle-resources-system, - cattle-sriov-system, - cattle-system, - cattle-ui-plugin-system, - cattle-windows-gmsa-system, - cert-manager, - cis-operator-system, - fleet-default, - ingress-nginx, - istio-system, - kube-node-lease, - kube-public, - kube-system, - longhorn-system, - rancher-alerting-drivers, - security-scan, - tigera-operator] - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - generate_serving_certificate: true -addons: | - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -```yaml -# 如果你打算在离线环境部署 Kubernetes， -# 请查阅文档以了解如何配置自定义的 RKE 镜像。 -nodes: [] -kubernetes_version: # 定义 RKE 版本 -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - generate_serving_certificate: true -addons: | - # Upstream Kubernetes restricted PSP policy - # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: restricted-noroot - spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - requiredDropCapabilities: - - ALL - # Allow core volume types. - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - # Require the container to run without root privileges. - rule: 'MustRunAsNonRoot' - seLinux: - # This policy assumes the nodes are using AppArmor rather than SELinux. - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - readOnlyRootFilesystem: false - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: psp:restricted-noroot - rules: - - apiGroups: - - extensions - resourceNames: - - restricted-noroot - resources: - - podsecuritypolicies - verbs: - - use - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: psp:restricted-noroot - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:restricted-noroot - subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:authenticated - --- - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -## 加固后的 RKE 集群模板配置参考 - -参考的 RKE 集群模板提供了实现 Kubernetes 加固安装所需的最低配置。RKE 模板用于提供 Kubernetes 并定义 Rancher 设置。有关安装 RKE 及其模板详情的其他信息，请参考 Rancher [文档](../../../../getting-started/installation-and-upgrade/installation-and-upgrade.md) 。 - - - - -```yaml -# -# 集群配置 -# -default_pod_security_admission_configuration_template_name: rancher-restricted -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # 定义集群名称 - -# -# Rancher 配置 -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # 定义 RKE 版本 - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: false - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -```yaml -# -# 集群配置 -# -default_pod_security_policy_template_id: restricted-noroot -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # 定义集群名称 - -# -# Rancher 配置 -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # 定义 RKE 版本 - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -## 结论 - -如果你按照本指南操作，由 Rancher 提供的 RKE 自定义集群将配置为通过 CIS Kubernetes Benchmark 测试。你可以查看我们的 RKE 自我评估指南，了解我们是如何验证每个 benchmarks 的，并且你可以在你的集群上执行相同的操作。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index cb3a548a8b1..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,2864 +0,0 @@ ---- -title: RKE 自我评估指南 - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - - - -本文档是 [RKE 加固指南](rke1-hardening-guide.md)的配套文档，该指南提供了关于如何加固正在生产环境中运行并由 Rancher 管理的 RKE 集群的指导方针。本 benchmark 指南可帮助你根据 CIS Kubernetes Benchmark 中的每个 control 来评估加固集群的安全性。 - -本指南对应以下版本的 Rancher、CIS Benchmarks 和 Kubernetes： - -| Rancher 版本 | CIS Benchmark 版本 | Kubernetes 版本 | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -本指南将介绍各种 controls，并提供更新的示例命令来审计 Rancher 创建的集群中的合规性。由于 Rancher 和 RKE 将 Kubernetes 服务安装为 Docker 容器，因此 CIS Kubernetes Benchmark 中的许多 control 验证检查不适用。这些检查将返回 `Not Applicable` 的结果。 - -本文档适用于 Rancher 运维人员、安全团队、审计员和决策者。 - -有关每个 control 的更多信息，包括详细描述和未通过测试的补救措施，请参考 CIS Kubernetes Benchmark v1.7 的相应部分。你可以在[互联网安全中心 (CIS)](https://www.cisecurity.org/benchmark/kubernetes/)创建免费账户后下载 benchmark。 - -## 测试方法 - -Rancher 和 RKE 通过 Docker 容器安装 Kubernetes 服务。配置是通过初始化时传递给容器的参数定义的，而不是通过配置文件。 - -在 control 审计与原始 CIS benchmark 不同时，提供了针对 Rancher 的特定审计命令以进行测试。在执行测试时，你将需要访问所有 RKE 节点主机上的命令行。这些命令还使用了 [kubectl](https://kubernetes.io/docs/tasks/tools/)（带有有效的配置文件）和 [jq](https://stedolan.github.io/jq/) 工具，在测试和评估测试结果时这些工具是必需的。 - -:::note - -本指南仅涵盖 `automated`（之前称为 `scored`）测试。 - -::: - -### Controls - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit:** - -```bash -stat -c %a /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'700' is equal to '700' -``` - -**Returned Value**: - -```console -700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd - -**Audit:** - -```bash -stat -c %U:%G /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'etcd:etcd' is present -``` - -**Returned Value**: - -```console -etcd:etcd -``` - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /etc/kubernetes/pki/ - -**Audit Script:** `check_files_owner_in_dir.sh` - -```bash -#!/usr/bin/env bash - -# This script is used to ensure the owner is set to root:root for -# the given directory and all the files in it -# -# inputs: -# $1 = /full/path/to/directory -# -# outputs: -# true/false - -INPUT_DIR=$1 - -if [[ "${INPUT_DIR}" == "" ]]; then - echo "false" - exit -fi - -if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then - echo "false" - exit -fi - -statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) -while read -r statInfoLine; do - f=$(echo ${statInfoLine} | cut -d' ' -f1) - p=$(echo ${statInfoLine} | cut -d' ' -f2) - - if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then - if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then - echo "false" - exit - fi - else - if [[ "$p" != "root:root" ]]; then - echo "false" - exit - fi - fi -done <<< "${statInfoLines}" - - -echo "true" -exit - -``` - -**Audit Execution:** - -```bash -./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= -When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'AlwaysPullImages' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-path' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxage' is greater or equal to 30 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxbackup' is greater or equal to 10 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxsize' is greater or equal to 100 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--encryption-provider-config' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. - -**Audit:** - -```bash -ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi -``` - -**Expected Result**: - -```console -'provider' is present -``` - -### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true -Cluster provisioned by RKE handles certificate rotation directly through RKE. - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--cert-file' is present AND '--key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---client-cert-auth="true" - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-cert-file' is present AND '--peer-key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---peer-client-cert-auth=true - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--trusted-ca-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Automated) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. -All configuration is passed in as arguments at container run time. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. - All configuration is passed in as arguments at container run time. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit:** - -```bash -stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit:** - -```bash -stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--event-qps' is greater or equal to 0 OR '--event-qps' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--rotate-certificates' is present OR '--rotate-certificates' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--pod-max-pids' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** pass - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -**Audit Script:** `check_for_default_sa.sh` - -```bash -#!/bin/bash - -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - -count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) -if [[ ${count_sa} -gt 0 ]]; then - echo "false" - exit -fi - -for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") -do - for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') - do - read kind name <<<$(IFS=","; echo $result) - resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) - if [[ ${resource_count} -gt 0 ]]; then - echo "false" - exit - fi - done -done - - -echo "true" - -``` - -**Audit Execution:** - -```bash -./check_for_default_sa.sh -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Manual) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/rancher-security.md b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/rancher-security.md index 85c1e15e37c..e2ed8a024a6 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/rancher-security.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/reference-guides/rancher-security/rancher-security.md @@ -67,7 +67,7 @@ Rancher 加固指南基于 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/api/v3-rancher-api-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/api/v3-rancher-api-guide.md index 89872d72845..a09d21eb6d7 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/api/v3-rancher-api-guide.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/api/v3-rancher-api-guide.md @@ -71,7 +71,7 @@ API 请求必须包含认证信息。认证是通过 [API 密钥](../reference-g ## 捕获 Rancher API 调用 -你可以使用浏览器开发人员工具来捕获 Rancher API 的调用方式。例如，你可以按照以下步骤使用 Chrome 开发人员工具来获取用于配置 RKE 集群的 API 调用： +You can use browser developer tools to capture how the v3 API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning a Rancher Kubernetes distribution cluster: 1. 在 Rancher UI 中，转到**集群管理**并单击**创建**。 1. 单击某个集群类型。此示例使用 Digital Ocean。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/contribute-to-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/contribute-to-rancher.md index baff7b735b9..ea433c1f2be 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/contribute-to-rancher.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/contribute-to-rancher.md @@ -35,7 +35,6 @@ title: 参与 Rancher 社区贡献 | (Rancher) Docker Machine | https://github.com/rancher/machine | 使用主机驱动时使用的 Docker Machine 二进制文件的源码仓库。这是 `docker/machine` 仓库的一个 fork。 | | machine-package | https://github.com/rancher/machine-package | 用于构建 Rancher Docker Machine 二进制文件。 | | kontainer-engine | https://github.com/rancher/kontainer-engine | kontainer-engine 的源码仓库，它是配置托管 Kubernetes 集群的工具。 | -| RKE repository | https://github.com/rancher/rke | Rancher Kubernetes Engine 的源码仓库，该工具可在任何主机上配置 Kubernetes 集群。 | | CLI | https://github.com/rancher/cli | Rancher 2.x 中使用的 Rancher CLI 的源码仓库。 | | (Rancher) Helm repository | https://github.com/rancher/helm | 打包的 Helm 二进制文件的源码仓库。这是 `helm/helm` 仓库的一个 fork。 | | Telemetry repository | https://github.com/rancher/telemetry | Telemetry 二进制文件的源码仓库。 | @@ -106,27 +105,6 @@ title: 参与 Rancher 社区贡献 -l app=rancher \ --timestamps=true ``` - - 在 RKE 集群的每个节点上使用 `docker` 的 Docker 安装 - - ``` - docker logs \ - --timestamps \ - $(docker ps | grep -E "rancher/rancher@|rancher_rancher" | awk '{ print $1 }') - ``` - - 使用 RKE 附加组件的 Kubernetes 安装 - - :::note - - 确保你配置了正确的 kubeconfig（例如，如果 Rancher Server 安装在 Kubernetes 集群上，则 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI 使用了嵌入式 kubectl。 - - ::: - - ``` - kubectl -n cattle-system \ - logs \ - --timestamps=true \ - -f $(kubectl --kubeconfig $KUBECONFIG get pods -n cattle-system -o json | jq -r '.items[] | select(.spec.containers[].name="cattle-server") | .metadata.name') - ``` - 系统日志记录（可能不存在，取决于操作系统） - `/var/log/messages` - `/var/log/syslog` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/container-network-interface-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/container-network-interface-providers.md index 0d9c93e18fc..15d7f94a841 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/container-network-interface-providers.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/container-network-interface-providers.md @@ -43,58 +43,9 @@ CNI 网络插件使用封装网络模型（例如 Virtual Extensible Lan，缩 ## Rancher 提供哪些 CNI 插件？ -### RKE Kubernetes 集群 - -Rancher 开箱即用地为 RKE Kubernetes 集群提供了几个 CNI 网络插件，分别是 Canal、Flannel、Calico 和 Weave。 - -如果你使用 Rancher 创建新的 Kubernetes 集群，你可以选择你的 CNI 网络插件。 - -#### Canal - -![Canal Logo](/img/canal-logo.png) - -Canal 是一个 CNI 网络插件，它很好地结合了 Flannel 和 Calico 的优点。它让你轻松地将 Calico 和 Flannel 网络部署为统一的网络解决方案，将 Calico 的网络策略执行与 Calico（未封装）和 Flannel（封装）丰富的网络连接选项结合起来。 - -Canal 是 Rancher 默认的 CNI 网络插件，并采用了 Flannel 和 VXLAN 封装。 - -Kubernetes Worker 需要打开 UDP 端口 `8472` (VXLAN) 和 TCP 端口 `9099`（健康检查）。如果使用 Wireguard，则需要打开 UDP 端口 `51820` 和 `51821`。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 - -![](/img/canal-diagram.png) - -有关详细信息，请参阅 [Canal GitHub 页面](https://github.com/projectcalico/canal)。 - -#### Flannel - -![Flannel Logo](/img/flannel-logo.png) - -Flannel 是为 Kubernetes 配置 L3 网络结构的简单方法。Flannel 在每台主机上运行一个名为 flanneld 的二进制 Agent，该 Agent 负责从更大的预配置地址空间中为每台主机分配子网租约。Flannel 通过 Kubernetes API 或直接使用 etcd 来存储网络配置、分配的子网、以及其他辅助数据（例如主机的公共 IP）。数据包使用某种后端机制来转发，默认封装为 [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)。 - -默认情况下，封装的流量是不加密的。Flannel 提供了两种加密方案： - -* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec)：使用 [strongSwan](https://www.strongswan.org/) 在 Kubernetes worker 之间建立加密的 IPSec 隧道。它是加密的实验性后端。 -* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard)：比 strongSwan 更快的替代方案。 - -Kubernetes Worker 需要打开 UDP 端口 `8472` (VXLAN)。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络要求)。 - -![Flannel Diagram](/img/flannel-diagram.png) - -有关详细信息，请参阅 [Flannel GitHub 页面](https://github.com/flannel-io/flannel)。 - -#### Weave - -![Weave Logo](/img/weave-logo.png) - -Weave 在云上的 Kubernetes 集群中启用网络和网络策略。此外，它还支持加密对等节点之间的流量。 - -Kubernetes worker 需要打开 TCP 端口 `6783`（控制端口）、UDP 端口 `6783` 和 UDP 端口 `6784`（数据端口）。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络要求)。 - -有关详细信息，请参阅以下页面： - -- [Weave Net 官网](https://github.com/weaveworks/weave/blob/master/site/overview.md) - ### RKE2 Kubernetes 集群 -Rancher 开箱即用地为 RKE2 Kubernetes 集群提供了几个 CNI 网络插件，分别是 [Canal](#canal)（见上一节）、Calico 和 Cilium。 +Rancher 开箱即用地为 RKE2 Kubernetes 集群提供了几个 CNI 网络插件，分别是 Canal、Calico 和 Cilium。 如果你使用 Rancher 创建新的 Kubernetes 集群，你可以选择你的 CNI 网络插件。 @@ -125,6 +76,20 @@ Calico 还提供了一种无状态的 IP-in-IP 或 VXLAN 封装模式。如果 - [Project Calico 官方网站](https://www.projectcalico.org/) - [Calico 项目 GitHub 页面](https://github.com/projectcalico/calico) +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal 是一个 CNI 网络插件，它很好地结合了 Flannel 和 Calico 的优点。它让你轻松地将 Calico 和 Flannel 网络部署为统一的网络解决方案，将 Calico 的网络策略执行与 Calico（未封装）和 Flannel（封装）丰富的网络连接选项结合起来。 + +Canal 是 Rancher 默认的 CNI 网络插件，并采用了 Flannel 和 VXLAN 封装。 + +Kubernetes Worker 需要打开 UDP 端口 `8472` (VXLAN) 和 TCP 端口 `9099`（健康检查）。如果使用 Wireguard，则需要打开 UDP 端口 `51820` 和 `51821`。有关详细信息，请参阅[下游集群的端口要求](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +![](/img/canal-diagram.png) + +有关详细信息，请参阅 [Canal GitHub 页面](https://github.com/projectcalico/canal) + #### Cilium ![Cilium Logo](/img/cilium-logo.png) @@ -201,4 +166,4 @@ Canal 是默认的 CNI 网络插件。对于大多数用例，我们推荐你使 ## 如何配置 CNI 网络插件？ -如需了解如何为你的集群配置网络插件，请参阅[集群选项](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。有关更高级的配置选项，请参阅有关使用[配置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群配置文件参考)和[网络插件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/)选项来配置集群的说明。 +如需了解如何为你的集群配置网络插件，请参阅[集群选项](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。有关更高级的配置选项，请参阅有关使用[配置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#rke-集群配置文件参考)和[网络插件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/)选项来配置集群的说明。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/deprecated-features.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/deprecated-features.md index e604fde7528..01fd6892692 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/deprecated-features.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/deprecated-features.md @@ -16,9 +16,7 @@ Rancher 将在 GitHub 上发布的 Rancher 的[发版说明](https://github.com/ | Patch 版本 | 发布时间 | | ----------------------------------------------------------------- | ------------------ | -| [2.11.2](https://github.com/rancher/rancher/releases/tag/v2.11.2) | 2025 年 5 月 22 日 | -| [2.11.1](https://github.com/rancher/rancher/releases/tag/v2.11.1) | 2025 年 4 月 24 日 | -| [2.11.0](https://github.com/rancher/rancher/releases/tag/v2.11.0) | 2025 年 3 月 31 日 | +| [2.12.0](https://github.com/rancher/rancher/releases/tag/v2.12.0) | 2025 年 7 月 30 日 | ## 当一个功能被标记为弃用我可以得到什么样的预期？ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/dockershim.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/dockershim.md deleted file mode 100644 index cb1d658b131..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/dockershim.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: Dockershim ---- - -Dockershim 是 Kubelet 和 Docker Daemon 之间的 CRI 兼容层。Kubernetes 1.20 版本宣布了[移除树内 Dockershim](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/)。目前计划在 Kubernetes 1.24 中移除。有关此移除的更多信息以及时间线，请参见 [Kubernetes Dockershim 弃用相关的常见问题](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 - -从 Kubernetes 1.21 开始。RKE 集群支持外部 Dockershim，来让用户继续使用 Docker 作为 CRI 运行时。现在，我们通过使用 [Mirantis 和 Docker ](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) 来确保 RKE 集群可以继续使用 Docker，从而实现上游开源社区的 Dockershim。 - -要启用外部 Dockershim，配置以下选项： - -``` -enable_cri_dockerd: true -``` - -如果你想使用其他容器运行时，Rancher 也提供使用 Containerd 作为默认运行时的，以边缘为中心的 K3s，和以数据中心为中心的 RKE2 Kubernetes 发行版。即使在 Kubernetes 1.24 删除了树内 Dockershim 之后，你也可以通过 Rancher 升级和管理导入的 RKE2 和 K3s Kubernetes 集群。 - -## 常见问题 - -
- -Q: 如果要获得 Rancher 对上游 Dockershim 的支持，我需要升级 Rancher 吗？ - -对于 RKE，Dockershim 的上游支持从 Kubernetes 1.21 开始。你需要使用 Rancher 2.6 或更高版本才能获取使用 Kubernetes 1.21 的 RKE 的支持。详情请参阅我们的[支持矩阵](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/)。 - -
- -Q: 我目前的 RKE 使用 Kubernetes 1.20。为了避免出现不再支持 Dockershim 的情况，我是否需要尽早将 RKE 升级到 Kubernetes 1.21？ - -A: 在使用 Kubernetes 1.20 的 RKE 中，Dockershim 版本依然可用，而且在 Kubernetes 1.24 之前不会在上游弃用。Kubernetes 会发出弃用 Dockershim 的警告，而 Rancher 在使用 Kubernetes 1.21 的 RKE 中已经缓解了这个问题。你可以按照计划正常升级到 Kubernetes 1.21，但也应该考虑在升级到 Kubernetes 1.22 时启用外部 Dockershim。在升级到 Kubernetes 1.24 之前，你需要启用外部 Dockershim，此时现有的实现都会被删除。 - -有关此移除的更多信息以及时间线，请参见 [Kubernetes Dockershim 弃用相关的常见问题](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 - -
- -Q: 如果我不想再依赖 Dockershim，我还有什么选择？ - -A: 你可以为 Kubernetes 使用不需要 Dockershim 支持的运行时，如 Containerd。RKE2 和 K3s 就是其中的两个选项。 - -
- -Q: 如果我目前使用 RKE1，但想切换到 RKE2，我可以怎样进行迁移？ - -A: Rancher 也在探索就地升级路径的可能性。此外，你始终可以使用 kubectl 将工作负载迁移到另一个集群。 - -
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/install-and-configure-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/install-and-configure-kubectl.md index 2b9764b84f2..656fcba1ce7 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/install-and-configure-kubectl.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/install-and-configure-kubectl.md @@ -10,17 +10,11 @@ title: 安装和配置 kubectl ## 配置 -使用 RKE 创建 Kubernetes 集群时，RKE 会在本地目录中创建一个 `kube_config_cluster.yml`，该文件包含使用 `kubectl` 或 `helm` 等工具连接到新集群的凭证。 - -你可以将此文件复制为 `$HOME/.kube/config`。如果你使用多个 Kubernetes 集群，将 `KUBECONFIG` 环境变量设置为 `kube_config_cluster.yml` 的路径： - -``` -export KUBECONFIG=$(pwd)/kube_config_cluster.yml -``` +When you create a Kubernetes cluster with RKE2/K3s, the Kubeconfig file is stored at `/etc/rancher/rke2/rke2.yaml` or `/etc/rancher/k3s/k3s.yaml` depending on your chosen distribution. These files are used to configure access to the Kubernetes cluster. 使用 `kubectl` 测试你的连接性，并查看你是否可以获取节点列表： -``` +```shell kubectl get nodes NAME STATUS ROLES AGE VERSION 165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/rancher-is-no-longer-needed.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/rancher-is-no-longer-needed.md index 0ae8e7be37b..48d4c445ffe 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/rancher-is-no-longer-needed.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/rancher-is-no-longer-needed.md @@ -12,9 +12,9 @@ title: 卸载 Rancher 如果删除了 Rancher，访问下游集群的方式取决于集群的类型和集群的创建方式。总而言之： -- **注册集群**：集群不受影响，你可以注册集群前的方法访问该集群。 +- **Registered/Imported clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. - **托管的 Kubernetes 集群**：如果你在 Kubernetes 云提供商（例如 EKS、GKE 或 AKS）中创建集群，你可以继续使用提供商的云凭证来管理集群。 -- **RKE 集群**：要访问 [RKE 集群](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md)，集群必须启用了[授权集群端点（authorized cluster endpoint，ACE）](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)，而且你必须从 Rancher UI 下载了集群的 kubeconfig 文件。RKE 集群默认启用授权集群端点。通过使用此端点，你可以直接使用 kubectl 访问你的集群，而不用通过 Rancher Server 的[认证代理](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-认证代理)进行通信。有关配置 kubectl 以使用授权集群端点的说明，请参阅[使用 kubectl 和 kubeconfig 文件直接访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验证)。这些集群将使用删除 Rancher 时配置的身份验证快照。 +- **Rancher provisioned clusters:** To access an [RKE2/K3s cluster](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. ## 如果我不想再使用 Rancher 了该怎么做？ @@ -51,7 +51,7 @@ title: 卸载 Rancher **结果**：注册的集群已与 Rancher 分离，并在 Rancher 外正常运行。 -## 如果我不想 Rancher 管理我的 RKE 集群或托管的 Kubernetes 集群该怎么办？ +## What if I don't want my hosted Kubernetes cluster managed by Rancher? 目前，我们没有将这些集群从 Rancher 中分离出来的功能。在这种情况下，“分离”指的是将 Rancher 组件移除出集群，并独立于 Rancher 管理对集群的访问。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/technical-items.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/technical-items.md index b781a8cebb1..d9c83604fc8 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/technical-items.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/faq/technical-items.md @@ -60,7 +60,7 @@ ClusterIP 是一个虚拟 IP，不会响应 ping。要测试 ClusterIP 是否配 ## Rancher 的状态存储在哪里？ - Docker 安装：在 `rancher/rancher` 容器的嵌入式 etcd 中，位于 `/var/lib/rancher`。 -- Kubernetes install：在为运行 Rancher 而创建的 RKE 集群的 etcd 中。 +- Kubernetes install: default location is in the `/var/lib/rancher/rke2` or `/var/lib/rancher/k3s` directories of the respective RKE2/K3s cluster created to run Rancher. ## 支持的 Docker 版本是如何确定的？ @@ -95,7 +95,7 @@ UI 由静态文件组成，并根据 API 的响应工作。换言之，UI 中可 ## 如何将其他参数/绑定/环境变量添加到 Rancher 启动的 Kubernetes 集群的 Kubernetes 组件中？ -你可以使用集群选项中的[配置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群配置文件参考)选项来添加其他参数/​​绑定/环境变量。有关详细信息，请参阅 RKE 文档中的[其他参数、绑定和环境变量](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/)，或浏览 [Cluster.ymls 示例](https://rancher.com/docs/rke/latest/en/example-yamls/)。 +You can add more arguments/binds/environment variables via the respective [RKE2 Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#cluster-configuration) or [K3s Config File](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md#cluster-configuration). ## 如何检查证书链是否有效？ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md index cc4e205c941..15eb0a7f299 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md @@ -60,6 +60,12 @@ RKE2 基于使用上游[集群 API](https://github.com/kubernetes-sigs/cluster-a - 完全在 Rancher 中配置 RKE2 集群 - 除了 Canal 之外，还可以选择 CNI 选项， Calico、Cilium 和 Multus +When you make changes to your cluster configuration in RKE2, this may result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. + +The following are some specific example configuration changes that may cause the described behavior: + +- When editing the cluster and enabling drain before delete, the existing control plane nodes and worker are deleted and new nodes are created. + RKE2 配置还包括在具有 Windows 节点的集群上安装 RKE2。 RKE2 的 Windows 功能包括： diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md deleted file mode 100644 index 57c3c855491..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: RKE1 和 RKE2 差​​异 ---- - - - -RKE2，也称为 RKE Government，是一个完全符合标准的 Kubernetes 发行版，它专注于安全性和合规性。它被认为是 Rancher Kubernetes Engine（即 RKE1）的下一个迭代。 - -RKE1 和 RKE2 有一些细微的差异，本文将重点介绍这些差异。 - -## controlplane 组件 - -RKE1 使用 Docker 来部署和管理 controlplane 组件，还使用 Docker 作为 Kubernetes 的容器运行时。相比之下，RKE2 将 controlplane 组件作为由 kubelet 管理的静态 pod 启动。RKE2 的容器运行时是 Containerd，它允许 Mirror 容器镜像仓库等内容。使用 Docker 的 RKE1 不允许 Mirror。 - -## Cluster API - -RKE2/K3s 配置是基于 Cluster API (CAPI) 上游框架之上构建的，这导致 RKE2 配置的集群的行为通常与 RKE1 配置的集群不同。 - -如果你在 RKE2 中更改集群配置，节点**可能**会重新配置。这是由 CAPI 控制器而不是 Rancher 控制的。请注意，etcd 节点不适用相同的行为。 - -以下是一些可能导致上述行为的配置更改示例： - -- 编辑集群并启用`删除前清空`时，会删除现有的 controlplane 节点和 worker 并创建新节点。 - -如果你是习惯于 RKE1 配置的用户，请注意新的 RKE2 行为。 - -## 名词解释 - -从 RKE1 到 RKE2，某些术语已更改或已不再使用。例如，在 RKE1中，你使用**节点模板**，而在 RKE2 中，你可以在创建或编辑集群时配置集群节点池。另一个例子是 RKE1 中的**节点池（node pool）** 现在在 RKE2 中称为**主机池（machine pool）**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md index 4532ddf2e7b..327e035803b 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -44,14 +44,13 @@ Rancher 会发现并显示由 `kubectl` 创建的资源。但是在发现资源 ## 直接使用下游集群进行身份验证 -本节旨在帮助你设置访问 [RKE 集群的替代方法](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md)。 +本节旨在帮助你设置访问 [Rancher-launched 集群的替代方法](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md)。 -此方法仅适用于启用了[授权集群端点](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)的 RKE、RKE2 和 K3s集群。在 Rancher 创建集群时，Rancher 会生成一个 kubeconfig 文件，其中包含用于访问集群的额外 kubectl 上下文。该上下文允许你使用 kubectl 通过下游集群进行身份验证，而无需通过 Rancher 进行身份验证。有关授权集群端点如何工作的详细说明，请参阅[此页面](authorized-cluster-endpoint.md)。 +此方法仅适用于启用了[授权集群端点](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)的 RKE2 和 K3s集群。在 Rancher 创建集群时，Rancher 会生成一个 kubeconfig 文件，其中包含用于访问集群的额外 kubectl 上下文。该上下文允许你使用 kubectl 通过下游集群进行身份验证，而无需通过 Rancher 进行身份验证。有关授权集群端点如何工作的详细说明，请参阅[此页面](authorized-cluster-endpoint.md)。 在 RKE2 和 K3s 集群上，你需要[手动启用](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#对-rke2-和-k3s-集群的授权集群端点支持)授权集群端点。 -我们的最佳实践是使用此方法来访问 RKE、RKE2 和 K3s集群。这样，万一你无法连接到 Rancher，你仍然可以访问该集群。 - +我们的最佳实践是使用此方法来访问 RKE2 和 K3s集群。这样，万一你无法连接到 Rancher，你仍然可以访问该集群。 :::note 先决条件： @@ -70,7 +69,7 @@ CURRENT NAME CLUSTER AUTHINFO N 在此示例中，当你将 `kubectl` 与第一个上下文 `my-cluster` 一起使用时，你将通过 Rancher Server 进行身份验证。 -使用第二个上下文 `my-cluster-controlplane-1`，你将使用授权集群端点进行身份验证，直接与下游 RKE 集群通信。 +使用第二个上下文 `my-cluster-controlplane-1`，你将使用授权集群端点进行身份验证，直接与下游 RKE2/K3s 集群通信。 我们建议使用具有授权集群端点的负载均衡器。有关详细信息，请参阅[推荐的架构](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#授权集群端点架构)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md index 369a7be6f29..a2d3d658629 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -118,66 +118,7 @@ $ sudo shutdown -r now ## 清理节点 - - - -在运行以下命令之前，首先通过 Rancher UI 删除节点。 - -要删除节点： - -1. 点击 **☰** 并选择**集群管理**。 -1. 在集群表中，单击节点所属集群的名称。 -1. 在第一个选项卡中，单击节点状态旁边的复选框。 -1. 单击**删除**。 - -要删除整个集群而不是单个节点，或者不使用 Rancher UI 重新移动节点，请按照以下步骤操作： - -1. 从节点中[删除](#docker-容器镜像和卷) Docker 容器并[卸载](#挂载)任何卷。 -1. 重启节点。 -1. [删除](#目录和文件)所有剩余的文件。 -1. 确认在重新启动后已正确清理网络接口和 IP 表。否则请再重启一次。 - -### Windows 节点 - -要清理 Windows 节点，请运行 `c:\\etc\\rancher` 中的脚本。此脚本删除 Kubernetes 生成的资源和执行二进制文件，还会删除防火墙规则和网络设置： - -``` -pushd c:\etc\rancher -.\cleanup.ps1 -popd -``` - -运行此脚本后，节点将重置并可以重新添加到 Kubernetes 集群。 - -### Docker 容器、镜像和卷 - -:::caution - -清理 Docker 容器时要小心。以下命令将删除节点上的*所有* Docker 容器、镜像和卷，包括与 Rancher 无关的容器： - -::: - -``` -docker rm -f $(docker ps -qa) -docker rmi -f $(docker images -q) -docker volume rm $(docker volume ls -q) -``` - -### 挂载 - -Kubernetes 组件和 secret 会留下以下挂载： - -* `/var/lib/kubelet` -* `/var/lib/rancher` -* `/var/lib/kubelet/pods/` 中的其他挂载 - -要卸载所有挂载，请运行： - -``` -for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done -``` - - + :::note @@ -245,53 +186,6 @@ sudo k3s-uninstall.sh ::: - - -| 目录 | -|------------------------------| -| `/etc/ceph` | -| `/etc/cni` | -| `/etc/kubernetes` | -| `/opt/cni` | -| `/opt/rke` | -| `/run/calico` | -| `/run/flannel` | -| `/run/secrets/kubernetes.io` | -| `/var/lib/calico` | -| `/var/lib/cni` | -| `/var/lib/etcd` | -| `/var/lib/kubelet` | -| `/var/lib/rancher/rke` | -| `/var/lib/weave` | -| `/var/log/containers` | -| `/var/log/kube-audit` | -| `/var/log/pods` | -| `/var/run/calico` | - -**清理目录**： - -```shell -rm -rf /etc/ceph \ - /etc/cni \ - /etc/kubernetes \ - /opt/cni \ - /opt/rke \ - /run/calico \ - /run/flannel \ - /run/secrets/kubernetes.io \ - /var/lib/calico \ - /var/lib/cni \ - /var/lib/etcd \ - /var/lib/kubelet \ - /var/lib/rancher/rke \ - /var/lib/weave \ - /var/log/containers \ - /var/log/kube-audit \ - /var/log/pods \ - /var/run/calico -``` - - | 目录 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md index be1431337b7..e231727abbd 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -12,20 +12,6 @@ title: 证书轮换 可以为以下服务轮换证书： - - - -- etcd -- kubelet（节点证书） -- kubelet（服务证书，如果[启用](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)） -- kube-apiserver -- kube-proxy -- kube-scheduler -- kube-controller-manager - - - - - admin - api-server - controller-manager @@ -38,9 +24,6 @@ title: 证书轮换 - kubelet - kube-proxy - - - :::note 如果你未轮换 webhook 证书，且证书用了一年后已经过期，请参阅此[页面](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md)。 @@ -64,15 +47,4 @@ Rancher 启动的 Kubernetes 集群能够通过 UI 轮换自动生成的证书 ## 补充说明 - - - -虽然 RKE CLI 可以为 Kubernetes 集群组件使用自定义证书，但 Rancher 目前不允许在 Rancher 启动的 Kubernetes 集群中上传这些证书。 - - - - -在 RKE2 中，etcd 和 controlplane 节点都被视为相同的 `server`。因此，如果你轮换其中一个组件的服务证书，则两者的证书都会被轮换。证书只会针对指定的服务更改，但你会看到两个组件的节点都进入更新状态。你可能还会看到仅 Worker 节点进入更新状态。这是在证书更改后重启 Worker，以确保他们获得最新的客户端证书。 - - - +在 RKE2/K3s 中，etcd 和 controlplane 节点都被视为相同的 `server`。因此，如果你轮换其中一个组件的服务证书，则两者的证书都会被轮换。证书只会针对指定的服务更改，但你会看到两个组件的节点都进入更新状态。你可能还会看到仅 Worker 节点进入更新状态。这是在证书更改后重启 Worker，以确保他们获得最新的客户端证书。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md index b8c52cf70c5..f05c3b9d4d9 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -2,39 +2,11 @@ title: 加密密钥轮换 --- -## RKE1 加密密钥轮换 +:::note 重要 -1. 使用以下两个选项之一来启用加密密钥轮换： +加密密钥轮换默认启用，不能禁用。 - - 在 Rancher UI 中的 **Cluster Options > Advanced Options > Secrets Encryption** 下选择 `Enabled` 单选按钮： - - ![启用加密密钥轮换](/img/rke1-enable-secrets-encryption.png) - - - 或者，应用以下 YAML： - - ```yaml - rancher_kubernetes_engine_config: - services: - kube_api: - secrets_encryption_config: - enabled: true - ``` - -2. 在 Rancher UI 中轮换密钥： - - 2.1. 点击 **☰ > 集群管理**。 - - 2.2. 在所选集群旁边的屏幕最右侧选择 **⋮ > Rotate Encryption Keys**： - - ![加密密钥轮换](/img/rke1-encryption-key.png) - - - -## RKE2 加密密钥轮换 - -_**v2.6.7 新功能**_ - -> **重要提示**：加密密钥轮换默认启用，不能禁用。 +::: 要在 Rancher UI 中轮换密钥： @@ -44,5 +16,4 @@ _**v2.6.7 新功能**_ ![加密密钥轮换](/img/rke2-encryption-key.png) - -> **注意**：有关 RKE2 密文加密配置的更多信息，请参阅 [RKE2 文档](https://docs.rke2.io/security/secrets_encryption)。 \ No newline at end of file +> **注意**：有关 RKE2 密文加密配置的更多信息，请参阅 [RKE2 文档](https://docs.rke2.io/security/secrets_encryption)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md index e883536a6ad..d9258538bfc 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -15,9 +15,7 @@ title: 安装 Adapter | Rancher 版本 | Adapter 版本 | |-----------------|:----------------:| -| v2.11.2 | v106.0.0+up6.0.0 | -| v2.11.1 | v106.0.0+up6.0.0 | -| v2.11.0 | v106.0.0+up6.0.0 | +| v2.12.0 | 107.0.0+up7.0.0 | ## 1. 获取对 Local 集群的访问权限 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/harvester/overview.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/harvester/overview.md index b5ee0ff89eb..0453eeb700d 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/harvester/overview.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/integrations-in-rancher/harvester/overview.md @@ -24,7 +24,7 @@ Harvester 功能开关用于管理对 Rancher 中虚拟化管理（VM）页面 ## Harvester 主机驱动 -[Harvester 主机驱动](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/)通常可用于 Rancher 中的 RKE 和 RKE2 选项。无论 Harvester 功能开关是否启用，主机驱动都是可用的。请注意，主机驱动默认处于关闭状态。用户只能通过集群管理页面在 Harvester 上创建 RKE 或 RKE2 集群。 +[Harvester 主机驱动](https://docs.harvesterhci.io/v1.5/rancher/node/node-driver/)通常可用于 Rancher 中的 K3s 和 RKE2 选项。无论 Harvester 功能开关是否启用，主机驱动都是可用的。请注意，主机驱动默认处于关闭状态。用户只能通过集群管理页面在 Harvester 上创建 K3s 或 RKE2 集群。 Harvester 允许通过 Harvester UI 上传和显示 `.ISO` 镜像，但 Rancher UI 是不支持的。这是因为 `.ISO` 镜像通常需要额外的设置，这会干扰干净的部署（即无需用户干预），并且它们通常不用于云环境。 @@ -32,11 +32,11 @@ Harvester 允许通过 Harvester UI 上传和显示 `.ISO` 镜像，但 Rancher ## 端口要求 -Harvester 集群的端口要求可以在[此处](https://docs.harvesterhci.io/v1.1/install/requirements#networking)找到。 +Harvester 集群的端口要求可以在[此处](https://docs.harvesterhci.io/v1.5/install/requirements#networking)找到。 此外，其他网络注意事项如下： - 请务必为 VM VLAN 网络启用物理交换机的 VLAN 中继端口。 -- 按照[此处](https://docs.harvesterhci.io/v1.1/networking/index)的网络设置指南进行操作。 +- 按照[此处](https://docs.harvesterhci.io/v1.5/networking/index)的网络设置指南进行操作。 -对于其他集群（例如 K3s 和 RKE1）的其他端口要求，请参阅[这些文档](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters)。 +对于其他集群（例如 K3s 和 RKE2）的其他端口要求，请参阅[这些文档](https://docs.harvesterhci.io/v1.5/install/requirements/#guest-clusters)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md index c3c9b7a732d..e0b9eb6757f 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md @@ -16,4 +16,4 @@ While a managed cluster is disconnected from Rancher, management operations will - **Cleaning Up Disconnected Clusters**: Regularly remove clusters that will no longer reconnect to Rancher (e.g., clusters that have been decommissioned or destroyed). Keeping such clusters in the Rancher management system consumes unnecessary resources, which could impact Rancher's performance over time. -- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE/RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. +- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index afc1247854f..64d58392251 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -26,9 +26,7 @@ Rancher Logging 使用的是 [Logging Operator](https://github.com/kube-logging/ ### Kubernetes 组件 -_ClusterFlows_ 能够收集 Kubernetes 集群中所有主机上所有容器的日志。如果这些容器包含在 Kubernetes Pod 中，这个方法是适用的。但是，RKE 容器不存在于 Kubernetes 内。 - -目前，Rancher 能搜集 RKE 容器的日志，但不能轻易过滤。这是因为这些日志不包含源容器的信息（例如 `etcd` 或 `kube-apiserver`）。 +_ClusterFlows_ 能够收集 Kubernetes 集群中所有主机上所有容器的日志。如果这些容器包含在 Kubernetes Pod 中，这个方法是适用的。 Rancher 的未来版本将包含源容器名称，来支持过滤这些组件的日志。该功能实现之后，你将能够自定义 _ClusterFlow_ 来**仅**检索 Kubernetes 组件日志，并将日志发送到适当的输出位置。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index df057a54983..3e7a33d3e88 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -18,10 +18,6 @@ title: Rancher 运行技巧 在部署节点时，请遵循 K8s 和 etcd 的最佳实践，其中包括禁用 swap，检查集群中的所有主机之间是否有良好的网络连接，为每个节点使用唯一的主机名、MAC 地址和 `product_uuids`，检查所需端口是否已经打开，并使用配置 SSD 的 etcd 进行部署。详情请参见 [kubernetes 官方文档](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin)和 [etcd 性能操作指南](https://etcd.io/docs/v3.5/op-guide/performance/)。 -## 使用 RKE 时：备份状态文件（Statefile） - -RKE 将集群状态记录在一个名为 `cluster.rkestate` 的文件中，该文件对集群的恢复和/或通过 RKE 维护集群非常重要。由于这个文件包含证书材料，我们强烈建议在备份前对该文件进行加密。请在每次运行 `rke up` 后备份状态文件。 - ## 在同一个数据中心运行集群中的所有节点 为达到最佳性能，请在同一地理数据中心运行所有三个节点。如果你在云（如 AWS）上运行节点，请在不同的可用区（AZ）中运行这三个节点。例如，在 us-west-2a 中运行节点 1，在 us-west-2b 中运行节点 2，在 us-west-2c 中运行节点 3。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md index 154b01c5ad4..68ae57eae46 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md @@ -76,7 +76,7 @@ Rancher 使用两个 Kubernetes 应用程序资源：`apps.projects.cattle.io` ### 使用授权集群端点 (ACE) -[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点) (ACE) 提供了 Rancher 部署的 RKE、RKE2 和 K3s 集群的 Kubernetes API 访问。启用后，ACE 会为生成的 `kubeconfig` 文件配置直接访问下游集群 Endpoint，从而绕过 Rancher 代理。在可以直接访问下游集群 Kubernetes API 的场景下，可以减少 Rancher 负载。有关更多信息，请参阅[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)配置说明。 +[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点) (ACE) 提供了 Rancher 部署的 RKE2 和 K3s 集群的 Kubernetes API 访问。启用后，ACE 会为生成的 `kubeconfig` 文件配置直接访问下游集群 Endpoint，从而绕过 Rancher 代理。在可以直接访问下游集群 Kubernetes API 的场景下，可以减少 Rancher 负载。有关更多信息，请参阅[授权集群端点](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)配置说明。 ### 减少 Event Handler 执行 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md index 21f3b31f422..8add5f0520a 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md @@ -14,7 +14,6 @@ title: 集群配置 集群配置选项取决于 Kubernetes 集群的类型： -- [RKE 集群配置](rancher-server-configuration/rke1-cluster-configuration.md) - [RKE2 集群配置](rancher-server-configuration/rke2-cluster-configuration.md) - [K3s 集群配置](rancher-server-configuration/k3s-cluster-configuration.md) - [EKS 集群配置](rancher-server-configuration/eks-cluster-configuration.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md index c3482cad893..684f10d8157 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md @@ -6,4 +6,4 @@ title: 下游集群配置 -以下文档将讨论[节点模板配置](node-template-configuration/node-template-configuration.md)和[主机配置](machine-configuration/machine-configuration.md)。 +以下文档将讨论[主机配置](machine-configuration/machine-configuration.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md index 4efbf1bb06b..9d42590068e 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md @@ -6,7 +6,6 @@ title: Rancher Server 配置 -- [RKE1 集群配置](rke1-cluster-configuration.md) - [RKE2 集群配置](rke2-cluster-configuration.md) - [K3s 集群配置](k3s-cluster-configuration.md) - [EKS 集群配置](eks-cluster-configuration.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/kubernetes-concepts.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/kubernetes-concepts.md index 5666ba5f4fc..6df41049db4 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/kubernetes-concepts.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/kubernetes-concepts.md @@ -6,13 +6,7 @@ title: Kubernetes 概念 ## 关于 Docker -Docker 是容器打包和运行时系统的标准。开发者在 Dockerfiles 中构建容器镜像，并在 Docker 镜像仓库中分发容器镜像。[Docker Hub](https://hub.docker.com) 是市面上主流的公有镜像仓库。许多企业还创建私有 Docker 镜像仓库。Docker 主要用于管理单个节点上的容器。 - -:::note - -由于 Kubernetes 已经成为了容器管理的主流工具，所以从 Rancher 2.x 版本开始，我们不再支持 Docker Swarm。如果你有使用 Docker 管理容器的需求，可以安装 Rancher 1.6 进行操作。 - -::: +Docker 是一个容器打包和运行时标准。开发者可以使用 Dockerfile 构建容器镜像，并从 Docker 镜像仓库分发容器镜像。[Docker Hub](https://hub.docker.com) 是一个流行的公共镜像仓库。许多组织也建立了私有 Docker 镜像仓库。Docker 主要用于管理单个节点上的容器。 ## 关于 Kubernetes @@ -22,7 +16,7 @@ Kubernetes 是容器和集群管理的标准。YAML 文件规定了组成一个 集群是可作为一个系统协同工作的一组计算机。 -_Kubernetes 集群_ 是使用 [Kubernetes 容器编排系统](https://kubernetes.io/)来部署、运维和伸缩 Docker 容器的集群，让你的组织实现应用自动化运维。 +_Kubernetes 集群_ 是使用 [Kubernetes 容器编排系统](https://kubernetes.io/)来部署、运维和伸缩 容器的集群，让你的组织实现应用自动化运维。 ## Kubernetes 集群中节点的角色 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md index 7e61817fbe6..2b518f95407 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -28,14 +28,6 @@ title: 架构推荐 ![运行 Rancher Management Server 的 K3s Kubernetes 集群的架构](/img/k3s-server-storage.svg) -### RKE Kubernetes 集群安装 - -在 RKE 安装中，集群数据在集群中的三个 etcd 节点上复制，以在某个节点发生故障时提供冗余和进行数据复制。 - -
运行 Rancher Management Server 的 RKE Kubernetes 集群的架构
- -![运行 Rancher Management Server 的 RKE Kubernetes 集群的架构](/img/rke-server-storage.svg) - ## Kubernetes 安装的负载均衡器推荐配置 我们建议你为负载均衡器和 Ingress Controller 使用以下配置： @@ -57,7 +49,7 @@ title: 架构推荐 ## Kubernetes 安装的推荐节点角色 -如果 Rancher 安装在 K3s Kubernetes 或 RKE Kubernetes 集群上，以下建议适用。 +如果 Rancher 安装在 K3s Kubernetes 上，则适用以下建议。 ### K3s 集群角色 @@ -65,38 +57,6 @@ title: 架构推荐 对于运行 Rancher Management Server 的集群，我们建议使用两个 server 节点。不需要 Agent 节点。 -### RKE 集群角色 - -如果 Rancher 安装在 RKE Kubernetes 集群上，该集群应具有三个节点，并且每个节点都应具有所有三个 Kubernetes 角色，分别是 etcd，controlplane 和 worker。 - -### Rancher Server 和下游 Kubernetes 集群的 RKE 集群架构对比 - -我们对 Rancher Server 集群上 RKE 节点角色建议，与对运行你的应用和服务的下游集群的建议相反。 - -在配置下游 Kubernetes 集群时，Rancher 使用 RKE 作为创建下游 Kubernetes 集群的工具。注意：Rancher 将在未来的版本中添加配置下游 K3s 集群的功能。 - -我们建议下游 Kubernetes 集群中的每个节点都只分配一个角色，以确保稳定性和可扩展性。 - -![Rancher Server 集群中和下游集群中节点的 Kubernetes 角色对比](/img/rancher-architecture-node-roles.svg) - -RKE 每个角色至少需要一个节点，但并不强制每个节点只能有一个角色。但是，我们建议为运行应用的集群中的每个节点，使用单独的角色，以保证在服务拓展时，worker 节点上的工作负载不影响 Kubernetes master 或集群的数据。 - -以下是我们对下游集群的最低配置建议： - -- **三个仅使用 etcd 角色的节点** ，以在三个节点中其中一个发生故障时，仍能保障集群的高可用性。 -- **两个只有 controlplane 角色的节点** ，以保证 master 组件的高可用性。 -- **一个或多个只有 worker 角色的节点**，用于运行 Kubernetes 节点组件，以及你部署的服务或应用的工作负载。 - -在设置 Rancher Server 时，在三个节点上使用全部这三个角色也是安全的，因为： - -* 它允许一个 `etcd` 节点故障。 -* 它通过多个 `controlplane` 节点来维护 master 组件的多个实例。 -* 此集群上没有创建除 Rancher 之外的其他工作负载。 - -由于 Rancher Server 集群中没有部署其他工作负载，因此在大多数情况下，这个集群都不需要使用我们出于可扩展性和可用性的考虑，而为下游集群推荐的架构。 - -有关下游集群的最佳实践，请查看[生产环境清单](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/checklist-for-production-ready-clusters.md)或[最佳实践](../best-practices/best-practices.md)。 - ## 授权集群端点架构 如果你使用[授权集群端点（ACE）](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授权集群端点)，我们建议你创建一个指向负载均衡器的 FQDN，这个负载均衡器把流量转到所有角色为 `controlplane` 的节点。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index 9d7570ba151..d08db8494df 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -39,7 +39,7 @@ Rancher 使用 [ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-c - 检测下游集群中的资源变化 - 将下游集群的当前状态变更到目标状态 - 配置集群和项目的访问控制策略 -- 通过调用所需的 Docker Machine 驱动和 Kubernetes 引擎（例如 RKE 和 GKE）来配置集群 +- 通过调用所需的 Docker Machine 驱动和 Kubernetes 引擎（例如，GKE）来配置集群 默认情况下，Cluster Controller 连接到 Cluster Agent，Rancher 才能与下游集群通信。如果 Cluster Agent 不可用，Cluster Controller 可以连接到 [Node Agent](#3-node-agents)。 @@ -60,7 +60,7 @@ Cluster Agent，也叫做 `cattle-cluster-agent`，是运行在下游集群中 授权集群端点（ACE）可连接到下游集群的 Kubernetes API Server，而不用通过 Rancher 认证代理调度请求。 -> 授权集群端点仅适用于 Rancher 启动的 Kubernetes 集群，即只适用于 Rancher [使用 RKE](../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) 来配置的集群。它不适用于导入的集群，也不适用于托管在 Kubernetes 提供商中的集群（例如 Amazon 的 EKS）。 +> 授权集群端点仅适用于 Rancher 启动的 Kubernetes 集群，即 [Rancher 配置的集群](../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) 它不适用于导入的集群，也不适用于托管在 Kubernetes 提供商中的集群（例如 Amazon 的 EKS）。 授权集群端点的主要用途： @@ -81,11 +81,9 @@ Cluster Agent，也叫做 `cattle-cluster-agent`，是运行在下游集群中 维护、排除问题和升级集群需要用到以下文件，请妥善保管这些文件： -- `rancher-cluster.yml`：RKE 集群配置文件。 -- `kube_config_rancher-cluster.yml`：集群的 Kubeconfig 文件，包含完全访问集群的凭证。如果 Rancher 出现问题时，你可以使用此文件认证由 Rancher 启动的 Kubernetes 集群。 -- `rancher-cluster.rkestate`：Kubernetes 集群状态文件，文件包含用于完全访问集群的凭证。注意：仅在使用 RKE v0.2.0 或更高版本时，才会创建此该文件。 -> **注意**：后两个文件名中的 `rancher-cluster` 部分取决于你命名 RKE 集群配置文件的方式。 +- `config.yaml`: The RKE2 and K3s cluster configuration file. +- `rke2.yaml` or `k3s.yaml`: The Kubeconfig file for your RKE2 or K3s cluster. This file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. 有关在没有 Rancher 认证代理和其他配置选项的情况下连接到集群的更多信息，请参见 [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 @@ -97,13 +95,7 @@ Rancher 使用什么工具配置下游集群，取决于集群的类型。 Rancher 可以动态启动云上（如 Amazon EC2、DigitalOcean、Azure 或 vSphere 等）的节点，然后在节点上安装 Kubernetes。 -Rancher 使用 [RKE](https://github.com/rancher/rke) 和 [docker-machine](https://github.com/rancher/machine) 来配置这类型的集群。 - -### Rancher 为自定义节点启动 Kubernetes - -在配置此类集群时，Rancher 会在现有节点上安装 Kubernetes，从而创建自定义集群。 - -Rancher 使用 [RKE](https://github.com/rancher/rke) 来启动此类集群。 +Rancher 使用 [docker-machine](https://github.com/rancher/machine) 来配置这类型的集群。 ### 托管的 Kubernetes 提供商 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md index 8dc764b2f9a..799268cdd1e 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md @@ -12,7 +12,6 @@ Rancher 为每个受支持的 Rancher 版本的 Kubernetes 发行版提供了特 Rancher 使用以下 Kubernetes 发行版： -- [**RKE**](https://rancher.com/docs/rke/latest/en/)（Rancher Kubernetes Engine）是经过 CNCF 认证的 Kubernetes 发行版，完全在 Docker 容器中运行。 - [**RKE2**](https://docs.rke2.io/) 是一个完全合规的 Kubernetes 发行版，专注于安全和合规性。 - [**K3s**](https://docs.k3s.io/) 是一个完全合规的，轻量级 Kubernetes 发行版。它易于安装，内存需求只有上游 Kubernetes 的一半，所有组件都在一个小于 100 MB 的二进制文件中。 @@ -22,12 +21,6 @@ Rancher 使用以下 Kubernetes 发行版： 每个自我评估指南都附有强化指南。这些指南与列出的 Rancher 版本一起进行了测试。每个自我评估指南都在特定的 Kubernetes 版本和 CIS Benchmark 版本上进行了测试。如果 CIS Benchmark 尚未针对你的 Kubernetes 版本进行验证，你可以使用现有指南，直到添加适合你的版本的指南。 -### RKE 指南 - -| Kubernetes 版本 | CIS Benchmark 版本 | 自我评估指南 | 加固指南 | -|--------------------|-----------------------|-----------------------|------------------| -| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [链接](rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [链接](rke1-hardening-guide/rke1-hardening-guide.md) | - ### RKE2 指南 | 类型 | Kubernetes 版本 | CIS Benchmark 版本 | 自我评估指南 | 加固指南 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md deleted file mode 100644 index eaffdb72d92..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md +++ /dev/null @@ -1,516 +0,0 @@ ---- -title: RKE 加固指南 ---- - - - - - - - -本文档提供了针对生产环境的 RKE 集群进行加固的具体指导，以便在使用 Rancher 部署之前进行配置。它概述了满足信息安全中心（Center for Information Security, CIS）Kubernetes benchmark controls 所需的配置和控制。 - -:::note -这份加固指南描述了如何确保你集群中的节点安全。我们建议你在安装 Kubernetes 之前遵循本指南。 -::: - -此加固指南适用于 RKE 集群，并与以下版本的 CIS Kubernetes Benchmark、Kubernetes 和 Rancher 相关联： - -| Rancher 版本 | CIS Benchmark 版本 | Kubernetes 版本 | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 至 v1.26 | - -:::note -- 在 Benchmark v1.24 及更高版本中，检查 id `4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated)` 可能会失败，因为 `/etc/kubernetes/ssl/kube-ca.pem` 默认设置为 644。 -- 在 Benchmark v1.7 中，不再需要 `--protect-kernel-defaults` (`4.2.6`) 参数，并已被 CIS 删除。 -::: - -有关如何评估加固的 RKE 集群与官方 CIS benchmark 的更多细节，请参考特定 Kubernetes 和 CIS benchmark 版本的 RKE 自我评估指南。 - -## 主机级别要求 - -### 配置 Kernel 运行时参数 - -建议对群集中的所有节点类型使用以下 `sysctl` 配置。在 `/etc/sysctl.d/90-kubelet.conf` 中设置以下参数： - -```ini -vm.overcommit_memory=1 -vm.panic_on_oom=0 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -运行 `sysctl -p /etc/sysctl.d/90-kubelet.conf` 以启用设置。 - -### 配置 `etcd` 用户和组 - -在安装 RKE 之前，需要设置 **etcd** 服务的用户帐户和组。 - -#### 创建 `etcd` 用户和组 - -要创建 **etcd** 用户和组，请运行以下控制台命令。 -下面的命令示例中使用 `52034` 作为 **uid** 和 **gid** 。 -任何有效且未使用的 **uid** 或 **gid** 都可以代替 `52034`。 - -```bash -groupadd --gid 52034 etcd -useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin -``` - -在通过集群配置文件 `config.yml` 部署RKE时，请更新 `etcd` 用户的 `uid` 和 `gid`： - -```yaml -services: - etcd: - gid: 52034 - uid: 52034 -``` - -## Kubernetes 运行时要求 - -### 配置 `default` Service Account - -#### 设置 `automountServiceAccountToken` 为 `false` 用于 `default` service accounts - -Kubernetes 提供了一个 default service account，供集群工作负载使用，其中没有为 pod 分配特定的 service account。 -如果需要从 pod 访问 Kubernetes API，则应为该 pod 创建特定的 service account，并向该 service account 授予权限。 -应配置 default service account，使其不提供 service account 令牌，并且不应具有任何明确的权限分配。 - -对于标准 RKE 安装上的每个命名空间（包括 `default` 和 `kube-system`），`default` service account 必须包含以下值： - -```yaml -automountServiceAccountToken: false -``` - -将以下配置保存到名为 `account_update.yaml` 的文件中。 - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -创建一个名为 `account_update.yaml` 的 bash 脚本文件。 -确保执行 `chmod +x account_update.sh` 命令，以赋予脚本执行权限。 - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -执行此脚本将 `account_update.yaml` 配置应用到所有命名空间中的 `default` service account。 - -### 配置网络策略 - -#### 确保所有命名空间都定义了网络策略 - -在同一个 Kubernetes 集群上运行不同的应用程序会带来风险，即某个受感染的应用程序可能会攻击相邻的应用程序。为确保容器只与其预期通信的容器进行通信，网络分段至关重要。网络策略规定了哪些 Pod 可以互相通信，以及与其他网络终端通信的方式。 - -网络策略是命名空间范围的。当在特定命名空间引入网络策略时，所有未被策略允许的流量将被拒绝。然而，如果在命名空间中没有网络策略，那么所有流量将被允许进入和离开该命名空间中的 Pod。要强制执行网络策略，必须启用容器网络接口（container network interface, CNI）插件。本指南使用 [Canal](https://github.com/projectcalico/canal) 来提供策略执行。有关 CNI 提供程序的其他信息可以在[这里](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/)找到。 - -一旦在集群上启用了 CNI 提供程序，就可以应用默认的网络策略。下面提供了一个 **permissive** 的示例供参考。如果你希望允许匹配某个命名空间中所有 Pod 的所有入站和出站流量（即使添加了策略导致某些 Pod 被视为”隔离”），你可以创建一个明确允许该命名空间中所有流量的策略。请将以下配置保存为 `default-allow-all.yaml`。有关网络策略的其他[文档](https://kubernetes.io/docs/concepts/services-networking/network-policies/)可以在 Kubernetes 站点上找到。 - -:::caution -此网络策略只是一个示例，不建议用于生产用途。 -::: - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-allow-all -spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress -``` - -创建一个名为 `apply_networkPolicy_to_all_ns.sh`的 Bash 脚本文件。 - -确保运行 `chmod +x apply_networkPolicy_to_all_ns.sh` 命令，以赋予脚本执行权限。 - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl apply -f default-allow-all.yaml -n ${namespace} -done -``` - -执行此脚本以将 `default-allow-all.yaml` 配置和 **permissive** 的 `NetworkPolicy` 应用于所有命名空间。 - -## 已知限制 - -- 当注册自定义节点仅提供公共 IP 时，Rancher **exec shell** 和 **查看 pod 日志** 在加固设置中**不起作用**。 此功能需要在注册自定义节点时提供私有 IP。 - -## 加固的 RKE `cluster.yml` 配置参考 - -参考的 `cluster.yml` 文件是由 RKE CLI 使用的，它提供了实现 RKE 加固安装所需的配置。 -RKE [文档](https://rancher.com/docs/rke/latest/en/installation/)提供了有关配置项的更多详细信息。这里参考的 `cluster.yml` 不包括必需的 `nodes` 指令，因为它取决于你的环境。在 RKE 中有关节点配置的文档可以在[这里](https://rancher.com/docs/rke/latest/en/config-options/nodes/)找到。 - -示例 `cluster.yml` 配置文件中包含了一个 Admission Configuration 策略，在 `services.kube-api.admission_configuration` 字段中指定。这个[示例](../../psa-restricted-exemptions.md)策略包含了命名空间的豁免规则，这对于在Rancher中正确运行导入的RKE集群非常必要，类似于Rancher预定义的 [`rancher-restricted`](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) 策略。 - -如果你希望使用 RKE 的默认 `restricted` 策略，则将 `services.kube-api.admission_configuration` 字段留空，并将 `services.pod_security_configuration` 设置为 `restricted`。你可以在 [RKE 文档](https://rke.docs.rancher.com/config-options/services/pod-security-admission)中找到更多信息。 - - - - -:::note -如果你打算将一个 RKE 集群导入到 Rancher 中，请参考此[文档](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md)以了解如何配置 PSA 以豁免 Rancher 系统命名空间。 -::: - -```yaml -# 如果你打算在离线环境部署 Kubernetes， -# 请查阅文档以了解如何配置自定义的 RKE 镜像。 -nodes: [] -kubernetes_version: # 定义 RKE 版本 -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - # 如果你在 `admission_configuration` 中设置了自定义策略， - # 请将 `pod_security_configuration` 字段留空。 - # 否则，将其设置为 `restricted` 以使用 RKE 预定义的受限策略， - # 并删除 `admission_configuration` 字段中的所有内容。 - # - # pod_security_configuration: restricted - # - admission_configuration: - apiVersion: apiserver.config.k8s.io/v1 - kind: AdmissionConfiguration - plugins: - - name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1 - kind: PodSecurityConfiguration - defaults: - enforce: "restricted" - enforce-version: "latest" - audit: "restricted" - audit-version: "latest" - warn: "restricted" - warn-version: "latest" - exemptions: - usernames: [] - runtimeClasses: [] - namespaces: [calico-apiserver, - calico-system, - cattle-alerting, - cattle-csp-adapter-system, - cattle-elemental-system, - cattle-epinio-system, - cattle-externalip-system, - cattle-fleet-local-system, - cattle-fleet-system, - cattle-gatekeeper-system, - cattle-global-data, - cattle-global-nt, - cattle-impersonation-system, - cattle-istio, - cattle-istio-system, - cattle-logging, - cattle-logging-system, - cattle-monitoring-system, - cattle-neuvector-system, - cattle-prometheus, - cattle-provisioning-capi-system, - cattle-resources-system, - cattle-sriov-system, - cattle-system, - cattle-ui-plugin-system, - cattle-windows-gmsa-system, - cert-manager, - cis-operator-system, - fleet-default, - ingress-nginx, - istio-system, - kube-node-lease, - kube-public, - kube-system, - longhorn-system, - rancher-alerting-drivers, - security-scan, - tigera-operator] - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - generate_serving_certificate: true -addons: | - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -```yaml -# 如果你打算在离线环境部署 Kubernetes， -# 请查阅文档以了解如何配置自定义的 RKE 镜像。 -nodes: [] -kubernetes_version: # 定义 RKE 版本 -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - generate_serving_certificate: true -addons: | - # Upstream Kubernetes restricted PSP policy - # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: restricted-noroot - spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - requiredDropCapabilities: - - ALL - # Allow core volume types. - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - # Require the container to run without root privileges. - rule: 'MustRunAsNonRoot' - seLinux: - # This policy assumes the nodes are using AppArmor rather than SELinux. - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - readOnlyRootFilesystem: false - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: psp:restricted-noroot - rules: - - apiGroups: - - extensions - resourceNames: - - restricted-noroot - resources: - - podsecuritypolicies - verbs: - - use - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: psp:restricted-noroot - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:restricted-noroot - subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:authenticated - --- - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -## 加固后的 RKE 集群模板配置参考 - -参考的 RKE 集群模板提供了实现 Kubernetes 加固安装所需的最低配置。RKE 模板用于提供 Kubernetes 并定义 Rancher 设置。有关安装 RKE 及其模板详情的其他信息，请参考 Rancher [文档](../../../../getting-started/installation-and-upgrade/installation-and-upgrade.md) 。 - - - - -```yaml -# -# 集群配置 -# -default_pod_security_admission_configuration_template_name: rancher-restricted -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # 定义集群名称 - -# -# Rancher 配置 -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # 定义 RKE 版本 - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: false - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -```yaml -# -# 集群配置 -# -default_pod_security_policy_template_id: restricted-noroot -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # 定义集群名称 - -# -# Rancher 配置 -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # 定义 RKE 版本 - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -## 结论 - -如果你按照本指南操作，由 Rancher 提供的 RKE 自定义集群将配置为通过 CIS Kubernetes Benchmark 测试。你可以查看我们的 RKE 自我评估指南，了解我们是如何验证每个 benchmarks 的，并且你可以在你的集群上执行相同的操作。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index cb3a548a8b1..00000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,2864 +0,0 @@ ---- -title: RKE 自我评估指南 - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - - - -本文档是 [RKE 加固指南](rke1-hardening-guide.md)的配套文档，该指南提供了关于如何加固正在生产环境中运行并由 Rancher 管理的 RKE 集群的指导方针。本 benchmark 指南可帮助你根据 CIS Kubernetes Benchmark 中的每个 control 来评估加固集群的安全性。 - -本指南对应以下版本的 Rancher、CIS Benchmarks 和 Kubernetes： - -| Rancher 版本 | CIS Benchmark 版本 | Kubernetes 版本 | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -本指南将介绍各种 controls，并提供更新的示例命令来审计 Rancher 创建的集群中的合规性。由于 Rancher 和 RKE 将 Kubernetes 服务安装为 Docker 容器，因此 CIS Kubernetes Benchmark 中的许多 control 验证检查不适用。这些检查将返回 `Not Applicable` 的结果。 - -本文档适用于 Rancher 运维人员、安全团队、审计员和决策者。 - -有关每个 control 的更多信息，包括详细描述和未通过测试的补救措施，请参考 CIS Kubernetes Benchmark v1.7 的相应部分。你可以在[互联网安全中心 (CIS)](https://www.cisecurity.org/benchmark/kubernetes/)创建免费账户后下载 benchmark。 - -## 测试方法 - -Rancher 和 RKE 通过 Docker 容器安装 Kubernetes 服务。配置是通过初始化时传递给容器的参数定义的，而不是通过配置文件。 - -在 control 审计与原始 CIS benchmark 不同时，提供了针对 Rancher 的特定审计命令以进行测试。在执行测试时，你将需要访问所有 RKE 节点主机上的命令行。这些命令还使用了 [kubectl](https://kubernetes.io/docs/tasks/tools/)（带有有效的配置文件）和 [jq](https://stedolan.github.io/jq/) 工具，在测试和评估测试结果时这些工具是必需的。 - -:::note - -本指南仅涵盖 `automated`（之前称为 `scored`）测试。 - -::: - -### Controls - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit:** - -```bash -stat -c %a /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'700' is equal to '700' -``` - -**Returned Value**: - -```console -700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd - -**Audit:** - -```bash -stat -c %U:%G /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'etcd:etcd' is present -``` - -**Returned Value**: - -```console -etcd:etcd -``` - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /etc/kubernetes/pki/ - -**Audit Script:** `check_files_owner_in_dir.sh` - -```bash -#!/usr/bin/env bash - -# This script is used to ensure the owner is set to root:root for -# the given directory and all the files in it -# -# inputs: -# $1 = /full/path/to/directory -# -# outputs: -# true/false - -INPUT_DIR=$1 - -if [[ "${INPUT_DIR}" == "" ]]; then - echo "false" - exit -fi - -if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then - echo "false" - exit -fi - -statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) -while read -r statInfoLine; do - f=$(echo ${statInfoLine} | cut -d' ' -f1) - p=$(echo ${statInfoLine} | cut -d' ' -f2) - - if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then - if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then - echo "false" - exit - fi - else - if [[ "$p" != "root:root" ]]; then - echo "false" - exit - fi - fi -done <<< "${statInfoLines}" - - -echo "true" -exit - -``` - -**Audit Execution:** - -```bash -./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= -When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'AlwaysPullImages' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-path' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxage' is greater or equal to 30 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxbackup' is greater or equal to 10 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxsize' is greater or equal to 100 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--encryption-provider-config' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. - -**Audit:** - -```bash -ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi -``` - -**Expected Result**: - -```console -'provider' is present -``` - -### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true -Cluster provisioned by RKE handles certificate rotation directly through RKE. - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--cert-file' is present AND '--key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---client-cert-auth="true" - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-cert-file' is present AND '--peer-key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---peer-client-cert-auth=true - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--trusted-ca-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Automated) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. -All configuration is passed in as arguments at container run time. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. - All configuration is passed in as arguments at container run time. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit:** - -```bash -stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit:** - -```bash -stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--event-qps' is greater or equal to 0 OR '--event-qps' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--rotate-certificates' is present OR '--rotate-certificates' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--pod-max-pids' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** pass - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -**Audit Script:** `check_for_default_sa.sh` - -```bash -#!/bin/bash - -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - -count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) -if [[ ${count_sa} -gt 0 ]]; then - echo "false" - exit -fi - -for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") -do - for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') - do - read kind name <<<$(IFS=","; echo $result) - resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) - if [[ ${resource_count} -gt 0 ]]; then - echo "false" - exit - fi - done -done - - -echo "true" - -``` - -**Audit Execution:** - -```bash -./check_for_default_sa.sh -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Manual) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/rancher-security.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/rancher-security.md index 85c1e15e37c..e2ed8a024a6 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/rancher-security.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.12/reference-guides/rancher-security/rancher-security.md @@ -67,7 +67,7 @@ Rancher 加固指南基于 -The following table summarizes different GitHub metrics to give you an idea of each project's popularity and activity levels. This data was collected in June 2025. +The following table summarizes different GitHub metrics to give you an idea of each project's popularity and activity levels. This data was collected in July 2025. | Provider | Project | Stars | Forks | Contributors | | ---- | ---- | ---- | ---- | ---- | | Canal | https://github.com/projectcalico/canal | 720 | 99 | 20 | -| Flannel | https://github.com/flannel-io/flannel | 9.2k | 2.9k | 239 | -| Calico | https://github.com/projectcalico/calico | 6.5k | 1.4k | 378 | +| Flannel | https://github.com/flannel-io/flannel | 9.2k | 2.9k | 242 | +| Calico | https://github.com/projectcalico/calico | 6.7k | 1.5k | 380 | | Weave | https://github.com/weaveworks/weave | 6.6k | 681 | 84 | -| Cilium | https://github.com/cilium/cilium | 21.9k | 3.3k | 948 | +| Cilium | https://github.com/cilium/cilium | 21.1k | 3.3k | 959 | diff --git a/shared-files/_glossary.md b/shared-files/_glossary.md index 45d0e6d73fc..91f8eb155ec 100644 --- a/shared-files/_glossary.md +++ b/shared-files/_glossary.md @@ -159,6 +159,21 @@

Related terms: RKE, RKE2

+
+ Kubeconfig +
+

+ Versions: Rancher v2.12.0 and later +

+
+ A Rancher resource kubeconfigs.ext.cattle.io used to generate kubeconfigs. +
+
+ kubeconfig +
+
+ The kubeconfig file is used to configure access to Kubernetes when used in conjunction with kubectl or other clients. The kubeconfig contents can be generated by creating a Kubeconfig resource. +
## L diff --git a/sidebars.js b/sidebars.js index 992fe3f5d7e..f38580109b8 100644 --- a/sidebars.js +++ b/sidebars.js @@ -18,26 +18,25 @@ const sidebars = { // But you can create a sidebar manually tutorialSidebar: [ - - 'rancher-manager', + "rancher-manager", { - type: 'category', - label: 'Getting Started', + type: "category", + label: "Getting Started", items: [ "getting-started/overview", { - type: 'category', - label: 'Quick Start Guides', + type: "category", + label: "Quick Start Guides", link: { - type: 'doc', + type: "doc", id: "getting-started/quick-start-guides/quick-start-guides", }, items: [ { - type: 'category', - label: 'Deploying Rancher Server', + type: "category", + label: "Deploying Rancher Server", link: { - type: 'doc', + type: "doc", id: "getting-started/quick-start-guides/deploy-rancher-manager/deploy-rancher-manager", }, items: [ @@ -52,63 +51,62 @@ const sidebars = { "getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal", "getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs", "getting-started/quick-start-guides/deploy-rancher-manager/helm-cli", - - ] + ], }, "getting-started/quick-start-guides/deploy-rancher-manager/prime", { - type: 'category', - label: 'Deploying Workloads', + type: "category", + label: "Deploying Workloads", link: { - type: 'doc', + type: "doc", id: "getting-started/quick-start-guides/deploy-workloads/deploy-workloads", }, items: [ "getting-started/quick-start-guides/deploy-workloads/workload-ingress", "getting-started/quick-start-guides/deploy-workloads/nodeports", - ] - } - ] + ], + }, + ], }, { - type: 'category', - label: 'Installation and Upgrade', + type: "category", + label: "Installation and Upgrade", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/installation-and-upgrade", }, items: [ { - type: 'category', - label: 'Installation Requirements', + type: "category", + label: "Installation Requirements", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/installation-requirements/installation-requirements", }, items: [ "getting-started/installation-and-upgrade/installation-requirements/install-docker", "getting-started/installation-and-upgrade/installation-requirements/dockershim", "getting-started/installation-and-upgrade/installation-requirements/port-requirements", - ] + ], }, { - type: 'category', - label: 'Installation References', + type: "category", + label: "Installation References", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/installation-references/installation-references", }, items: [ "getting-started/installation-and-upgrade/installation-references/helm-chart-options", "getting-started/installation-and-upgrade/installation-references/tls-settings", - "getting-started/installation-and-upgrade/installation-references/feature-flags" - ] + "getting-started/installation-and-upgrade/installation-references/feature-flags", + ], }, { - type: 'category', - label: 'Install/Upgrade on a Kubernetes Cluster', + type: "category", + label: "Install/Upgrade on a Kubernetes Cluster", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/install-upgrade-on-a-kubernetes-cluster", }, items: [ @@ -119,21 +117,21 @@ const sidebars = { "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks", "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke", "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting", - ] + ], }, { - type: 'category', - label: 'Other Installation Methods', + type: "category", + label: "Other Installation Methods", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/other-installation-methods/other-installation-methods", }, items: [ { - type: 'category', - label: 'Air-Gapped Helm CLI Install', + type: "category", + label: "Air-Gapped Helm CLI Install", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/air-gapped-helm-cli-install", }, items: [ @@ -142,41 +140,41 @@ const sidebars = { "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes", "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha", "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands", - ] + ], }, { - type: 'category', - label: 'Rancher on a Single Node with Docker', + type: "category", + label: "Rancher on a Single Node with Docker", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/rancher-on-a-single-node-with-docker", }, items: [ "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher", "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher", "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting", - ] + ], }, { - type: 'category', - label: 'Rancher Behind an HTTP Proxy', + type: "category", + label: "Rancher Behind an HTTP Proxy", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/rancher-behind-an-http-proxy", }, items: [ "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure", "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes", "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher", - ] - } + ], + }, ], }, { - type: 'category', - label: 'Resources', + type: "category", + label: "Resources", link: { - type: 'doc', + type: "doc", id: "getting-started/installation-and-upgrade/resources/resources", }, items: [ @@ -188,40 +186,40 @@ const sidebars = { "getting-started/installation-and-upgrade/resources/update-rancher-certificate", "getting-started/installation-and-upgrade/resources/bootstrap-password", "getting-started/installation-and-upgrade/resources/local-system-charts", - ] + ], }, "getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes", "getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher", - ] - } - ] + ], + }, + ], }, { - type: 'category', - label: 'How-to Guides', + type: "category", + label: "How-to Guides", items: [ { - type: 'category', - label: 'New User Guides', + type: "category", + label: "New User Guides", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/new-user-guides", }, items: [ { - type: 'category', - label: 'Authentication, Permissions, and Global Configuration', + type: "category", + label: "Authentication, Permissions, and Global Configuration", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-permissions-and-global-configuration", }, items: [ { - type: 'category', - label: 'Configuring Authentication', + type: "category", + label: "Configuring Authentication", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/authentication-config", }, items: [ @@ -237,47 +235,48 @@ const sidebars = { "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-generic-oidc", - ] + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito", + ], }, { - type: 'category', - label: 'Configuring OpenLDAP', + type: "category", + label: "Configuring OpenLDAP", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/configure-openldap", }, items: [ "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference", - ] + ], }, { - type: 'category', - label: 'Configuring Microsoft AD Federation Service (SAML)', + type: "category", + label: "Configuring Microsoft AD Federation Service (SAML)", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-microsoft-ad-federation-service-saml", }, items: [ "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs", - ] + ], }, { - type: 'category', - label: 'Configuring Shibboleth (SAML)', + type: "category", + label: "Configuring Shibboleth (SAML)", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/configure-shibboleth-saml", }, items: [ "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions", - ] + ], }, { - type: 'category', - label: 'Managing Role-Based Access Control (RBAC)', + type: "category", + label: "Managing Role-Based Access Control (RBAC)", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/manage-role-based-access-control-rbac", }, items: [ @@ -285,26 +284,26 @@ const sidebars = { "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles", - ] + ], }, "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/jwt-authentication", { - type: 'category', - label: 'About Provisioning Drivers', + type: "category", + label: "About Provisioning Drivers", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers", }, items: [ "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers", - ] + ], }, { - type: 'category', - label: 'About RKE1 Templates', + type: "category", + label: "About RKE1 Templates", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/about-rke1-templates", }, items: [ @@ -316,51 +315,52 @@ const sidebars = { "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases", - ] + ], }, "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding", - ] + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center" + ], }, { - type: 'category', - label: 'Cluster Administration', + type: "category", + label: "Cluster Administration", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/manage-clusters/manage-clusters", }, items: [ { - type: 'category', - label: 'Access Clusters', + type: "category", + label: "Access Clusters", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/manage-clusters/access-clusters/access-clusters", }, items: [ "how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig", "how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint", "how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters", - ] + ], }, { - type: 'category', - label: 'Install Cluster Autoscaler', + type: "category", + label: "Install Cluster Autoscaler", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/install-cluster-autoscaler", }, items: [ "how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups", - ] + ], }, { - type: 'category', - label: 'Create Kubernetes Persistent Storage', + type: "category", + label: "Create Kubernetes Persistent Storage", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/create-kubernetes-persistent-storage", }, items: [ @@ -370,20 +370,20 @@ const sidebars = { "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver", "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes", "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes", - ] + ], }, { - type: 'category', - label: 'Provisioning Storage Examples', + type: "category", + label: "Provisioning Storage Examples", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/provisioning-storage-examples", }, items: [ "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs", "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage", "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage", - ] + ], }, "how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces", @@ -396,13 +396,14 @@ const sidebars = { "how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools", "how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes", - ] + + ], }, { - type: 'category', - label: 'Setting up a Kubernetes Cluster for Rancher Server', + type: "category", + label: "Setting up a Kubernetes Cluster for Rancher Server", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-cluster-setup/kubernetes-cluster-setup", }, items: [ @@ -413,10 +414,10 @@ const sidebars = { ], }, { - type: 'category', - label: 'Infrastructure Setup', + type: "category", + label: "Infrastructure Setup", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/infrastructure-setup/infrastructure-setup", }, items: [ @@ -430,19 +431,19 @@ const sidebars = { ], }, { - type: 'category', - label: 'Kubernetes Clusters in Rancher Setup', + type: "category", + label: "Kubernetes Clusters in Rancher Setup", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/kubernetes-clusters-in-rancher-setup", }, items: [ "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters", { - type: 'category', - label: 'Checklist for Production-Ready Clusters', + type: "category", + label: "Checklist for Production-Ready Clusters", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/checklist-for-production-ready-clusters", }, items: [ @@ -451,10 +452,10 @@ const sidebars = { ], }, { - type: 'category', - label: 'Setting up Clusters from Hosted Kubernetes Providers', + type: "category", + label: "Setting up Clusters from Hosted Kubernetes Providers", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/set-up-clusters-from-hosted-kubernetes-providers", }, items: [ @@ -464,27 +465,27 @@ const sidebars = { "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei", - ] + ], }, { - type: 'category', - label: 'Launching Kubernetes on Windows Clusters', + type: "category", + label: "Launching Kubernetes on Windows Clusters", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/use-windows-clusters", }, items: [ "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway", - "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance" - ] + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance", + ], }, { - type: 'category', - label: 'Setting up Cloud Providers', + type: "category", + label: "Setting up Cloud Providers", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/set-up-cloud-providers", }, items: [ @@ -493,33 +494,33 @@ const sidebars = { "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere", - ] + ], }, { - type: 'category', - label: 'Migrate to an Out-of-tree Cloud Provider', + type: "category", + label: "Migrate to an Out-of-tree Cloud Provider", items: [ "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-amazon", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-vsphere", "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/migrate-to-an-out-of-tree-cloud-provider/migrate-to-out-of-tree-azure", - ] + ], }, "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters", - ] + ], }, { - type: 'category', - label: 'Launching Kubernetes with Rancher', + type: "category", + label: "Launching Kubernetes with Rancher", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher", }, items: [ { - type: 'category', - label: 'Launching New Nodes in an Infra Provider', + type: "category", + label: "Launching New Nodes in an Infra Provider", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/use-new-nodes-in-an-infra-provider", }, items: [ @@ -528,51 +529,51 @@ const sidebars = { "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster", "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster", + + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster", { - type: 'category', - label: 'Creating a VMware vSphere Cluster', + type: "category", + label: "Creating a VMware vSphere Cluster", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere", }, items: [ "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere", "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials", "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template", - "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm" - ] + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm", + ], }, { - type: 'category', - label: 'Creating a Nutanix AOS Cluster', + type: "category", + label: "Creating a Nutanix AOS Cluster", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix", }, items: [ "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos", - ] - } - ] + ], + }, + ], }, - "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences", - "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents", - ] + ], }, { - type: 'category', - label: 'Kubernetes Resources Setup', + type: "category", + label: "Kubernetes Resources Setup", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-resources-setup", }, items: [ { - type: 'category', - label: 'Workloads and Pods', + type: "category", + label: "Workloads and Pods", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/workloads-and-pods", }, items: [ @@ -580,13 +581,13 @@ const sidebars = { "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads", "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads", "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar", - ] + ], }, { - type: 'category', - label: 'Horizontal Pod Autoscaler', + type: "category", + label: "Horizontal Pod Autoscaler", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/horizontal-pod-autoscaler", }, items: [ @@ -594,20 +595,20 @@ const sidebars = { "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui", "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl", "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl", - ] + ], }, { - type: 'category', - label: 'Load Balancer and Ingress Controller', + type: "category", + label: "Load Balancer and Ingress Controller", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/load-balancer-and-ingress-controller", }, items: [ "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing", "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses", "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration", - ] + ], }, "how-to-guides/new-user-guides/kubernetes-resources-setup/create-services", @@ -618,25 +619,25 @@ const sidebars = { "how-to-guides/new-user-guides/kubernetes-resources-setup/secrets", "how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries", - ] + ], }, { - type: 'category', - label: 'Helm Charts and Apps', + type: "category", + label: "Helm Charts and Apps", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/helm-charts-in-rancher/helm-charts-in-rancher", }, items: [ "how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps", - "how-to-guides/new-user-guides/helm-charts-in-rancher/oci-repositories" - ] + "how-to-guides/new-user-guides/helm-charts-in-rancher/oci-repositories", + ], }, { - type: 'category', - label: 'Backup, Restore, and Disaster Recovery', + type: "category", + label: "Backup, Restore, and Disaster Recovery", link: { - type: 'doc', + type: "doc", id: "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery", }, items: [ @@ -648,34 +649,34 @@ const sidebars = { "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher", "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters", "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup", - ] + ], }, "how-to-guides/new-user-guides/add-users-to-projects", "how-to-guides/new-user-guides/manage-namespaces", - ] + ], }, { - type: 'category', - label: 'Advanced User Guides', + type: "category", + label: "Advanced User Guides", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/advanced-user-guides", }, items: [ { - type: 'category', - label: 'Project Administration', + type: "category", + label: "Project Administration", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/manage-projects/manage-projects", }, items: [ { - type: 'category', - label: 'Project Resource Quotas', + type: "category", + label: "Project Resource Quotas", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/manage-project-resource-quotas", }, items: [ @@ -683,15 +684,15 @@ const sidebars = { "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces", "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits", "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types", - ] - } - ] + ], + }, + ], }, { - type: 'category', - label: 'Monitoring/Alerting Guides', + type: "category", + label: "Monitoring/Alerting Guides", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/monitoring-alerting-guides/monitoring-alerting-guides", }, items: [ @@ -702,10 +703,10 @@ const sidebars = { "how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard", "how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage", { - type: 'category', - label: 'Prometheus Federator Guides', + type: "category", + label: "Prometheus Federator Guides", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/prometheus-federator-guides", }, items: [ @@ -713,39 +714,39 @@ const sidebars = { "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator", "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards", "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads", - "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors" - ] - } - ] + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors", + ], + }, + ], }, { - type: 'category', - label: 'Monitoring Configuration Guides', + type: "category", + label: "Monitoring Configuration Guides", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/monitoring-v2-configuration-guides", }, items: [ { - type: 'category', - label: 'Advanced Configuration', + type: "category", + label: "Advanced Configuration", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/advanced-configuration", }, items: [ "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager", "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus", "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules", - ] - } - ] + ], + }, + ], }, { - type: 'category', - label: 'Istio Setup Guides', + type: "category", + label: "Istio Setup Guides", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/istio-setup-guide/istio-setup-guide", }, items: [ @@ -755,42 +756,40 @@ const sidebars = { "how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway", "how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management", "how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic", - ] + ], }, { - type: 'category', - label: 'CIS Scan Guides', + type: "category", + label: "Compliance Scan Guides", link: { - type: 'doc', - id: "how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides", + type: "doc", + id: "how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides", }, items: [ - "how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark", - "how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark", - "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan", - "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule", - "how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests", - "how-to-guides/advanced-user-guides/cis-scan-guides/view-reports", - "how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark", - "how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", - "how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run", - ] + "how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance", + "how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance", + "how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan", + "how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule", + "how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports", + "how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance", + "how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", + "how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run", + ], }, { - type: 'category', - label: 'Enabling Experimental Features', + type: "category", + label: "Enabling Experimental Features", link: { - type: 'doc', + type: "doc", id: "how-to-guides/advanced-user-guides/enable-experimental-features/enable-experimental-features", }, items: [ - "how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination", "how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64", "how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers", "how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features", "how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery", "how-to-guides/advanced-user-guides/enable-experimental-features/cluster-role-aggregation", - ] + ], }, "how-to-guides/advanced-user-guides/open-ports-with-firewalld", "how-to-guides/advanced-user-guides/tune-etcd-for-large-installs", @@ -799,41 +798,43 @@ const sidebars = { "how-to-guides/advanced-user-guides/enable-user-retention", "how-to-guides/advanced-user-guides/enable-cluster-agent-scheduling-customization", "how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer", - ] - } - ] + "how-to-guides/advanced-user-guides/configure-oidc-provider", + "how-to-guides/advanced-user-guides/ui-server-side-pagination" + ], + }, + ], }, { - type: 'category', - label: 'Reference Guides', + type: "category", + label: "Reference Guides", items: [ { - type: 'category', - label: 'Best Practice Guides', + type: "category", + label: "Best Practice Guides", link: { - type: 'doc', + type: "doc", id: "reference-guides/best-practices/best-practices", }, items: [ { - type: 'category', - label: 'Rancher Server', + type: "category", + label: "Rancher Server", link: { - type: 'doc', + type: "doc", id: "reference-guides/best-practices/rancher-server/rancher-server", }, items: [ "reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere", "reference-guides/best-practices/rancher-server/rancher-deployment-strategy", "reference-guides/best-practices/rancher-server/tips-for-running-rancher", - "reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale" - ] + "reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale", + ], }, { - type: 'category', - label: 'Rancher-Managed Clusters', + type: "category", + label: "Rancher-Managed Clusters", link: { - type: 'doc', + type: "doc", id: "reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters", }, items: [ @@ -841,37 +842,37 @@ const sidebars = { "reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices", "reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers", "reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere", - "reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters" - ] - } - ] + "reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters", + ], + }, + ], }, { - type: 'category', - label: 'Rancher Architecture', + type: "category", + label: "Rancher Architecture", link: { - type: 'doc', + type: "doc", id: "reference-guides/rancher-manager-architecture/rancher-manager-architecture", }, items: [ "reference-guides/rancher-manager-architecture/rancher-server-and-components", "reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters", "reference-guides/rancher-manager-architecture/architecture-recommendations", - ] + ], }, { - type: 'category', - label: 'Cluster Configuration', + type: "category", + label: "Cluster Configuration", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/cluster-configuration", }, items: [ { - type: 'category', - label: 'Rancher Server Configuration', + type: "category", + label: "Rancher Server Configuration", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration", }, items: [ @@ -881,43 +882,43 @@ const sidebars = { "reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration", "reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration", { - type: 'category', - label: 'GKE Cluster Configuration Reference', + type: "category", + label: "GKE Cluster Configuration Reference", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-cluster-configuration", }, items: [ "reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters", - ] + ], }, { - type: 'category', - label: 'Use Existing Nodes', + type: "category", + label: "Use Existing Nodes", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/use-existing-nodes", }, items: [ "reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options", - ] + ], }, "reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters", - ] + ], }, { - type: 'category', - label: 'Downstream Cluster Configuration', + type: "category", + label: "Downstream Cluster Configuration", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration", }, items: [ { - type: 'category', - label: 'Node Template Configuration', + type: "category", + label: "Node Template Configuration", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/node-template-configuration", }, items: [ @@ -926,19 +927,20 @@ const sidebars = { "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure", "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere", "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix", - ] + ], }, { - type: 'category', - label: 'Machine Configuration', + type: "category", + label: "Machine Configuration", link: { - type: 'doc', + type: "doc", id: "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/machine-configuration", }, items: [ "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2", "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean", "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure", + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce" ] } ] @@ -946,22 +948,22 @@ const sidebars = { ] }, { - type: 'category', - label: 'Single-Node Rancher in Docker', + type: "category", + label: "Single-Node Rancher in Docker", link: { - type: 'doc', + type: "doc", id: "reference-guides/single-node-rancher-in-docker/single-node-rancher-in-docker", }, items: [ "reference-guides/single-node-rancher-in-docker/http-proxy-configuration", "reference-guides/single-node-rancher-in-docker/advanced-options", - ] + ], }, { - type: 'category', - label: 'Backup & Restore Configuration', + type: "category", + label: "Backup & Restore Configuration", link: { - type: 'doc', + type: "doc", id: "reference-guides/backup-restore-configuration/backup-restore-configuration", }, items: [ @@ -973,10 +975,10 @@ const sidebars = { }, "reference-guides/kubernetes-concepts", { - type: 'category', - label: 'Monitoring Configuration Reference', + type: "category", + label: "Monitoring Configuration Reference", link: { - type: 'doc', + type: "doc", id: "reference-guides/monitoring-v2-configuration/monitoring-v2-configuration", }, items: [ @@ -988,21 +990,19 @@ const sidebars = { ], }, { - type: 'category', - label: 'Prometheus Federator', + type: "category", + label: "Prometheus Federator", link: { - type: 'doc', + type: "doc", id: "reference-guides/prometheus-federator/prometheus-federator", }, - items: [ - "reference-guides/prometheus-federator/rbac", - ] + items: ["reference-guides/prometheus-federator/rbac"], }, { - type: 'category', - label: 'User Settings', + type: "category", + label: "User Settings", link: { - type: 'doc', + type: "doc", id: "reference-guides/user-settings/user-settings", }, items: [ @@ -1013,16 +1013,16 @@ const sidebars = { ], }, { - type: 'category', - label: 'CLI with Rancher', + type: "category", + label: "CLI with Rancher", link: { - type: 'doc', + type: "doc", id: "reference-guides/cli-with-rancher/cli-with-rancher", }, items: [ "reference-guides/cli-with-rancher/rancher-cli", "reference-guides/cli-with-rancher/kubectl-utility", - ] + ], }, "reference-guides/rancher-cluster-tools", @@ -1033,61 +1033,18 @@ const sidebars = { "reference-guides/rke1-template-example-yaml", "reference-guides/rancher-webhook", { - type: 'category', - label: 'Rancher Security Guides', + type: "category", + label: "Rancher Security Guides", link: { - type: 'doc', + type: "doc", id: "reference-guides/rancher-security/rancher-security", }, items: [ { - type: 'category', - label: 'Hardening Guides', + type: "category", + label: "SELinux RPM", link: { - type: 'doc', - id: "reference-guides/rancher-security/hardening-guides/hardening-guides", - }, - items: [ - { - type: 'category', - label: 'RKE Hardening Guides', - link: { - type: 'doc', - id: "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide", - }, - items: [ - "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", - ], - }, - { - type: 'category', - label: 'RKE2 Hardening Guides', - link: { - type: 'doc', - id: "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide", - }, - items: [ - "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", - ], - }, - { - type: 'category', - label: 'K3s Hardening Guides', - link: { - type: 'doc', - id: "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide", - }, - items: [ - "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", - ], - }, - ], - }, - { - type: 'category', - label: 'SELinux RPM', - link: { - type: 'doc', + type: "doc", id: "reference-guides/rancher-security/selinux-rpm/selinux-rpm", }, items: [ @@ -1099,126 +1056,117 @@ const sidebars = { "reference-guides/rancher-security/rancher-security-best-practices", "reference-guides/rancher-security/security-advisories-and-cves", "reference-guides/rancher-security/psa-restricted-exemptions", - "reference-guides/rancher-security/rancher-webhook-hardening" + "reference-guides/rancher-security/rancher-webhook-hardening", ], - } - ] + }, + ], }, { - "type": "category", - "label": "Integrations in Rancher", - "link": { - "type": "doc", - "id": "integrations-in-rancher/integrations-in-rancher" + type: "category", + label: "Integrations in Rancher", + link: { + type: "doc", + id: "integrations-in-rancher/integrations-in-rancher", }, - "items": [ + items: [ "integrations-in-rancher/kubernetes-distributions/kubernetes-distributions", { - "type": "category", - "label": "Virtualization on Kubernetes with Harvester", - "link": { - "type": "doc", - "id": "integrations-in-rancher/harvester/harvester" + type: "category", + label: "Virtualization on Kubernetes with Harvester", + link: { + type: "doc", + id: "integrations-in-rancher/harvester/harvester", }, - "items": [ - "integrations-in-rancher/harvester/overview" - ] + items: ["integrations-in-rancher/harvester/overview"], }, { - "type": "category", - "label": "Cloud Native Storage with Longhorn", - "link": { - "type": "doc", - "id": "integrations-in-rancher/longhorn/longhorn" + type: "category", + label: "Cloud Native Storage with Longhorn", + link: { + type: "doc", + id: "integrations-in-rancher/longhorn/longhorn", }, - "items": [ - "integrations-in-rancher/longhorn/overview" - ] + items: ["integrations-in-rancher/longhorn/overview"], }, { - "type": "category", - "label": "Container Security with Neuvector", - "link": { - "type": "doc", - "id": "integrations-in-rancher/neuvector/neuvector" + type: "category", + label: "Container Security with Neuvector", + link: { + type: "doc", + id: "integrations-in-rancher/neuvector/neuvector", }, - "items": [ - "integrations-in-rancher/neuvector/overview" - ] + items: ["integrations-in-rancher/neuvector/overview"], }, "integrations-in-rancher/suse-observability/suse-observability", "integrations-in-rancher/kubewarden/kubewarden", "integrations-in-rancher/elemental/elemental", { - "type": "category", - "label": "Continuous Delivery with Fleet", - "link": { - "type": "doc", - "id": "integrations-in-rancher/fleet/fleet" + type: "category", + label: "Continuous Delivery with Fleet", + link: { + type: "doc", + id: "integrations-in-rancher/fleet/fleet", }, - "items": [ + items: [ "integrations-in-rancher/fleet/overview", "integrations-in-rancher/fleet/architecture", "integrations-in-rancher/fleet/windows-support", - "integrations-in-rancher/fleet/use-fleet-behind-a-proxy" - ] - }, - { - "type": "category", - "label": "Cluster API (CAPI) with Rancher Turtles", - "link": { - "type": "doc", - "id": "integrations-in-rancher/cluster-api/cluster-api" - }, - "items": [ - "integrations-in-rancher/cluster-api/overview" - ] - }, - "integrations-in-rancher/rancher-desktop", - { - type: 'category', - label: 'Cloud Marketplace Integration', - link: { - type: 'doc', - id: "integrations-in-rancher/cloud-marketplace/cloud-marketplace" - }, - items: [ - { - type: 'category', - label: 'AWS Marketplace Integration', - link: { - type: 'doc', - id: "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/aws-cloud-marketplace" - }, - items: [ - 'integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements', - 'integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter', - 'integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter', - 'integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues' - ] - }, - 'integrations-in-rancher/cloud-marketplace/supportconfig' - ] - }, - { - type: 'category', - label: 'CIS Scans', - link: { - type: 'doc', - id: "integrations-in-rancher/cis-scans/cis-scans", - }, - items: [ - "integrations-in-rancher/cis-scans/configuration-reference", - "integrations-in-rancher/cis-scans/rbac-for-cis-scans", - "integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests", - "integrations-in-rancher/cis-scans/custom-benchmark", + "integrations-in-rancher/fleet/use-fleet-behind-a-proxy", ], }, { - type: 'category', - label: 'Istio', + type: "category", + label: "Cluster API (CAPI) with Rancher Turtles", link: { - type: 'doc', + type: "doc", + id: "integrations-in-rancher/cluster-api/cluster-api", + }, + items: ["integrations-in-rancher/cluster-api/overview"], + }, + "integrations-in-rancher/rancher-desktop", + { + type: "category", + label: "Cloud Marketplace Integration", + link: { + type: "doc", + id: "integrations-in-rancher/cloud-marketplace/cloud-marketplace", + }, + items: [ + { + type: "category", + label: "AWS Marketplace Integration", + link: { + type: "doc", + id: "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/aws-cloud-marketplace", + }, + items: [ + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues", + ], + }, + "integrations-in-rancher/cloud-marketplace/supportconfig", + ], + }, + { + type: "category", + label: "Compliance Scans", + link: { + type: "doc", + id: "integrations-in-rancher/compliance-scans/compliance-scans", + }, + items: [ + "integrations-in-rancher/compliance-scans/configuration-reference", + "integrations-in-rancher/compliance-scans/rbac-for-compliance-scans", + "integrations-in-rancher/compliance-scans/custom-benchmark", + ], + }, + { + type: "category", + label: "Istio", + link: { + type: "doc", id: "integrations-in-rancher/istio/istio", }, items: [ @@ -1226,25 +1174,25 @@ const sidebars = { "integrations-in-rancher/istio/rbac-for-istio", "integrations-in-rancher/istio/disable-istio", { - type: 'category', - label: 'Configuration Options', + type: "category", + label: "Configuration Options", link: { - type: 'doc', + type: "doc", id: "integrations-in-rancher/istio/configuration-options/configuration-options", }, items: [ "integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations", "integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster", "integrations-in-rancher/istio/configuration-options/project-network-isolation", - ] - } - ] + ], + }, + ], }, { - type: 'category', - label: 'Logging', + type: "category", + label: "Logging", link: { - type: 'doc', + type: "doc", id: "integrations-in-rancher/logging/logging", }, items: [ @@ -1253,22 +1201,22 @@ const sidebars = { "integrations-in-rancher/logging/logging-helm-chart-options", "integrations-in-rancher/logging/taints-and-tolerations", { - type: 'category', - label: 'Custom Resource Configuration', + type: "category", + label: "Custom Resource Configuration", link: { - type: 'doc', + type: "doc", id: "integrations-in-rancher/logging/custom-resource-configuration/custom-resource-configuration", }, items: [ "integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows", - "integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs" - ] - } - ] + "integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs", + ], + }, + ], }, { - type: 'category', - label: 'Monitoring and Alerting', + type: "category", + label: "Monitoring and Alerting", link: { type: "doc", id: "integrations-in-rancher/monitoring-and-alerting/monitoring-and-alerting", @@ -1279,35 +1227,34 @@ const sidebars = { "integrations-in-rancher/monitoring-and-alerting/built-in-dashboards", "integrations-in-rancher/monitoring-and-alerting/windows-support", "integrations-in-rancher/monitoring-and-alerting/promql-expressions", - ] + ], }, "integrations-in-rancher/rancher-extensions", - ] + ], }, { - type: 'category', - label: 'FAQ', + type: "category", + label: "FAQ", items: [ "faq/general-faq", "faq/deprecated-features", "faq/install-and-configure-kubectl", - "faq/dockershim", "faq/technical-items", "faq/security", "faq/container-network-interface-providers", "faq/rancher-is-no-longer-needed", - ] + ], }, { - type: 'category', - label: 'Troubleshooting', + type: "category", + label: "Troubleshooting", items: [ "troubleshooting/general-troubleshooting", { - type: 'category', - label: 'Kubernetes Components', + type: "category", + label: "Kubernetes Components", link: { - type: 'doc', + type: "doc", id: "troubleshooting/kubernetes-components/kubernetes-components", }, items: [ @@ -1315,11 +1262,11 @@ const sidebars = { "troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes", "troubleshooting/kubernetes-components/troubleshooting-nginx-proxy", "troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components", - ] + ], }, { - type: 'category', - label: 'Other Troubleshooting Tips', + type: "category", + label: "Other Troubleshooting Tips", items: [ "troubleshooting/other-troubleshooting-tips/kubernetes-resources", "troubleshooting/other-troubleshooting-tips/networking", @@ -1329,30 +1276,30 @@ const sidebars = { "troubleshooting/other-troubleshooting-tips/logging", "troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs", "troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation", - ] - } - ] + ], + }, + ], }, { - "type": "category", - "label": "Rancher Kubernetes API", - "items": [ + type: "category", + label: "Rancher Kubernetes API", + items: [ "api/quickstart", { - "type": "category", - "label": "Example Workflows", - "items": [ - "api/workflows/projects" - ] + type: "category", + label: "Example Workflows", + items: ["api/workflows/projects", + "api/workflows/kubeconfigs", + "api/workflows/tokens"], }, "api/api-reference", "api/api-tokens", "api/extension-apiserver", "api/v3-rancher-api-guide", - ] + ], }, "contribute-to-rancher", "glossary", - ] -} + ], +}; module.exports = sidebars; diff --git a/src/pages/versions.md b/src/pages/versions.md index 9e4039a8ade..cb3140abeed 100644 --- a/src/pages/versions.md +++ b/src/pages/versions.md @@ -5,6 +5,27 @@ title: Rancher Documentation Versions ### Current Versions +Here you can find links to supporting documentation for the current released version of Rancher v2.12, and its availability for [Rancher Prime](/v2.12/getting-started/quick-start-guides/deploy-rancher-manager/prime) and the Community version of Rancher: + + + + + + + + + + + + + + + + + + +
VersionDocumentationRelease NotesSupport MatrixPrimeCommunity
v2.12.0DocumentationRelease Notes
N/A
N/A
✓
+ Here you can find links to supporting documentation for the current released version of Rancher v2.11, and its availability for [Rancher Prime](/v2.11/getting-started/quick-start-guides/deploy-rancher-manager/prime) and the Community version of Rancher: @@ -17,12 +38,12 @@ Here you can find links to supporting documentation for the current released ver - + - + - +
Community
v2.11.3v2.11.4 DocumentationRelease NotesRelease Notes
N/A
✓
✓
NA
@@ -38,9 +59,9 @@ Here you can find links to supporting documentation for the current released ver Community - v2.10.7 + v2.10.8 Documentation - Release Notes + Release Notes
N/A
✓
N/A
@@ -102,6 +123,14 @@ Here you can find links to supporting documentation for previous versions of Ran Prime Community + + v2.11.3 + Documentation + Release Notes +
Support Matrix
+
✓
+
✓
+ v2.11.2 Documentation @@ -139,6 +168,14 @@ Here you can find links to supporting documentation for previous versions of Ran Prime Community + + v2.10.7 + Documentation + Release Notes +
Support Matrix
+
✓
+
N/A
+ v2.10.6 Documentation diff --git a/versioned_docs/version-2.10/faq/deprecated-features.md b/versioned_docs/version-2.10/faq/deprecated-features.md index 81984e43a7a..8a47612fca0 100644 --- a/versioned_docs/version-2.10/faq/deprecated-features.md +++ b/versioned_docs/version-2.10/faq/deprecated-features.md @@ -16,6 +16,7 @@ Rancher will publish deprecated features as part of the [release notes](https:// | Patch Version | Release Date | |---------------|---------------| +| [2.10.8](https://github.com/rancher/rancher/releases/tag/v2.10.8) | July 30, 2025 | | [2.10.7](https://github.com/rancher/rancher/releases/tag/v2.10.7) | June 25, 2025 | | [2.10.6](https://github.com/rancher/rancher/releases/tag/v2.10.6) | May 22, 2025 | | [2.10.5](https://github.com/rancher/rancher/releases/tag/v2.10.5) | Apr 24, 2025 | diff --git a/versioned_docs/version-2.10/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/versioned_docs/version-2.10/getting-started/installation-and-upgrade/installation-references/feature-flags.md index 969252b53e9..1ccb25756bd 100644 --- a/versioned_docs/version-2.10/getting-started/installation-and-upgrade/installation-references/feature-flags.md +++ b/versioned_docs/version-2.10/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -24,7 +24,19 @@ The following is a list of feature flags available in Rancher. If you've upgrade - `harvester`: Manages access to the Virtualization Management page, where users can navigate directly to Harvester clusters and access the Harvester UI. See [Harvester Integration Overview](../../../integrations-in-rancher/harvester/overview.md) for more information. - `istio-virtual-service-ui`: Enables a [visual interface](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md) to create, read, update, and delete Istio virtual services and destination rules, which are Istio traffic management features. - `legacy`: Enables a set of features from 2.5.x and earlier, that are slowly being phased out in favor of newer implementations. These are a mix of deprecated features as well as features that will eventually be available to newer versions. This flag is disabled by default on new Rancher installations. If you're upgrading from a previous version of Rancher, this flag is enabled. -- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream RKE2/K3s clusters, currently limited to imported clusters and the local cluster, with plans to expand support to node-driver clusters. +- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream imported RKE2/K3s clusters, as well as in the local cluster if it is an RKE2/K3s cluster. + +:::note Important: + +This `managed-system-upgrade-controller` flag is intended for **internal use only** and does not have an associated Feature CR. Use with caution. + +To control whether Rancher should manage the Kubernetes version of imported RKE2/K3s clusters, it is recommended to use the **imported-cluster-version-management** feature that is available in Rancher v2.11.0 or newer. + +For more details, please refer to the Rancher documentation for version v2.11 or newer. + +::: + + - `multi-cluster-management`: Allows multi-cluster provisioning and management of Kubernetes clusters. This flag can only be set at install time. It can't be enabled or disabled later. - `rke1-custom-node-cleanup`: Enables cleanup of deleted RKE1 custom nodes. We recommend that you keep this flag enabled, to prevent removed nodes from attempting to rejoin the cluster. - `rke2`: Enables provisioning RKE2 clusters. This flag is enabled by default. diff --git a/versioned_docs/version-2.10/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/versioned_docs/version-2.10/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md index 3e33e726550..4e0834b1511 100644 --- a/versioned_docs/version-2.10/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md +++ b/versioned_docs/version-2.10/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -19,6 +19,7 @@ In order to deploy and run the adapter successfully, you need to ensure its vers | Rancher Version | Adapter Version | |-----------------|------------------| +| v2.10.8 | v105.0.0+up5.0.1 | | v2.10.7 | v105.0.0+up5.0.1 | | v2.10.6 | v105.0.0+up5.0.1 | | v2.10.5 | v105.0.0+up5.0.1 | diff --git a/versioned_docs/version-2.10/reference-guides/rancher-webhook.md b/versioned_docs/version-2.10/reference-guides/rancher-webhook.md index bbfe2df7cdf..8a2379b77fa 100644 --- a/versioned_docs/version-2.10/reference-guides/rancher-webhook.md +++ b/versioned_docs/version-2.10/reference-guides/rancher-webhook.md @@ -20,6 +20,7 @@ Each Rancher version is designed to be compatible with a single version of the w | Rancher Version | Webhook Version | Availability in Prime | Availability in Community | |-----------------|-----------------|-----------------------|---------------------------| +| v2.10.8 | v0.6.9 | ✓ | ✗ | | v2.10.7 | v0.6.8 | ✓ | ✗ | | v2.10.6 | v0.6.7 | ✓ | ✗ | | v2.10.5 | v0.6.6 | ✓ | ✗ | diff --git a/versioned_docs/version-2.11/faq/deprecated-features.md b/versioned_docs/version-2.11/faq/deprecated-features.md index 959e3edf1d2..be867750779 100644 --- a/versioned_docs/version-2.11/faq/deprecated-features.md +++ b/versioned_docs/version-2.11/faq/deprecated-features.md @@ -16,6 +16,7 @@ Rancher will publish deprecated features as part of the [release notes](https:// | Patch Version | Release Date | |---------------|---------------| +| [2.11.4](https://github.com/rancher/rancher/releases/tag/v2.11.4) | July 30, 2025 | | [2.11.3](https://github.com/rancher/rancher/releases/tag/v2.11.3) | June 25, 2025 | | [2.11.2](https://github.com/rancher/rancher/releases/tag/v2.11.2) | May 22, 2025 | | [2.11.1](https://github.com/rancher/rancher/releases/tag/v2.11.1) | Apr 24, 2025 | diff --git a/versioned_docs/version-2.11/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/versioned_docs/version-2.11/getting-started/installation-and-upgrade/installation-references/feature-flags.md index ce153bce599..7f1093c718c 100644 --- a/versioned_docs/version-2.11/getting-started/installation-and-upgrade/installation-references/feature-flags.md +++ b/versioned_docs/version-2.11/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -26,7 +26,30 @@ The following is a list of feature flags available in Rancher. If you've upgrade - `imperative-api-extension`: Enables Rancher's [extension API server](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) to register new APIs to Kubernetes. This flag is enabled by default. See the [Extension API Server](../../../api/extension-apiserver.md) page for more information. - `istio-virtual-service-ui`: Enables a [visual interface](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md) to create, read, update, and delete Istio virtual services and destination rules, which are Istio traffic management features. - `legacy`: Enables a set of features from 2.5.x and earlier, that are slowly being phased out in favor of newer implementations. These are a mix of deprecated features as well as features that will eventually be available to newer versions. This flag is disabled by default on new Rancher installations. If you're upgrading from a previous version of Rancher, this flag is enabled. -- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream RKE2/K3s clusters, currently limited to imported clusters and the local cluster, with plans to expand support to node-driver clusters. +- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream imported RKE2/K3s clusters, as well as in the local cluster if it is an RKE2/K3s cluster. + +:::note Important: + +This `managed-system-upgrade-controller` flag is intended for **internal use only** and does not have an associated Feature CR. Use with caution. + +To control whether Rancher should manage the Kubernetes version of imported RKE2/K3s clusters, it is recommended to use the [imported-cluster-version-management](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters) feature that is available in Rancher v2.11.0 or newer. + +::: + +:::danger + +If the `managed-system-upgrade-controller` flag was **disabled** in Rancher v2.10.x, and any imported RKE2/K3s clusters were upgraded **outside of Rancher**, follow the steps below to prevent the unexpected installation of the system-upgrade-controller app and to ensure the [imported-cluster-version-management](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters) feature works correctly: + +1. Upgrade Rancher to v2.11.0 or newer, making sure to **retain** the `managed-system-upgrade-controller=false` feature flag in Helm values if it was set during the v2.10.x installation. +1. After Rancher is fully up and running, disable the `imported-cluster-version-management` setting. You can do this either through the Rancher UI by clicking **☰ > Global Settings > Settings > imported-cluster-version-management**, or by editing the corresponding `Setting.management.cattle.io/v3` custom resource via kubectl. +1. Perform a second Helm upgrade, this time omitting the `managed-system-upgrade-controller=false` feature flag. + +Now, the imported cluster version management is disabled by default, and Rancher no longer installs the system-upgrade-controller app on imported clusters automatically. + +You can enable this feature on a per-cluster basis. For more information, please refer to the [documentation](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters). + +::: + - `multi-cluster-management`: Allows multi-cluster provisioning and management of Kubernetes clusters. This flag can only be set at install time. It can't be enabled or disabled later. - `rke1-custom-node-cleanup`: Enables cleanup of deleted RKE1 custom nodes. We recommend that you keep this flag enabled, to prevent removed nodes from attempting to rejoin the cluster. - `rke2`: Enables provisioning RKE2 clusters. This flag is enabled by default. diff --git a/versioned_docs/version-2.11/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/versioned_docs/version-2.11/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md index c4618be475f..fbdc5c0356e 100644 --- a/versioned_docs/version-2.11/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md +++ b/versioned_docs/version-2.11/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -53,6 +53,10 @@ You can obtain `` and `` by loggi ## Upgrade +:::danger +Rancher upgrades to version 2.12.0 and later will be blocked if any RKE1-related resources are detected, as the Rancher Kubernetes Engine (RKE/RKE1) is end of life as of **July 31, 2025**. For detailed cleanup and recovery steps, refer to the [RKE1 Resource Validation and Upgrade Requirements in Rancher v2.12](#rke1-resource-validation-and-upgrade-requirements-in-rancher-v212). +::: + During upgrade, you create a copy of the data from your current Rancher container and a backup in case something goes wrong. Then you deploy the new version of Rancher in a new container using your existing data. ### 1. Create a copy of the data from your Rancher server container @@ -388,6 +392,45 @@ See [Restoring Cluster Networking](https://github.com/rancher/rancher-docs/tree/ Remove the previous Rancher server container. If you only stop the previous Rancher server container (and don't remove it), the container may restart after the next server reboot. +## RKE1 Resource Validation and Upgrade Requirements in Rancher v2.12 + +Rancher v2.12.0 and later has removed support for the Rancher Kubernetes Engine (RKE/RKE1). During upgrade, Rancher validates the cluster resources and blocks the upgrade if any RKE1-related resources are detected. + +This validation affects the following resource types: + +- Clusters with `rkeConfig` (`clusters.management.cattle.io`) +- NodeTemplates (`nodetemplates.management.cattle.io`) +- ClusterTemplates (`clustertemplates.management.cattle.io`) + +This is particularly relevant for single-node Docker installations, where Rancher is not running during the upgrade. In such cases, controllers are not available to automatically clean up deprecated resources, and the upgrade process will fail early with an error listing the blocking resources. + +### 1. Pre-Upgrade (Recommended) + +Before upgrading, while Rancher is still running: + +- Run the `pre-upgrade-hook` cleanup script to delete all RKE1 clusters and templates. You can find the script in the Rancher GitHub repository: [pre-upgrade-hook.sh](https://github.com/rancher/rancher/blob/v2.12.0/chart/scripts/pre-upgrade-hook.sh). +- This allows Rancher to clean up associated resources and finalizers. + +### 2. Post-Upgrade Failure Due to Residual RKE1 Resources + +If the upgrade to Rancher v2.12.0 or later is attempted without prior cleanup of RKE1 resources: + +- The upgrade will fail and display an error listing the resource names that are preventing the upgrade. +- This occurs because Rancher includes validation to detect and block upgrades when unsupported RKE1 resources are still present. +- To proceed, [rollback](#rolling-back) to the previous Rancher version, delete the identified resources, and then retry after [manual cleanup](#manual-cleanup-after-rollback). + +:::note Helm-based Rancher +Helm-based Rancher installations are not affected by this issue, as Rancher remains available during the upgrade and can perform resource cleanup as needed. +::: + +### Manual Cleanup After Rollback + +Users should perform the following steps after rolling back to a previous Rancher version: + +- **Manually delete** the resources listed in the upgrade error message (e.g., RKE1 clusters, NodeTemplates, ClusterTemplates). +- If deletion is blocked due to **finalizers**, edit the resources and remove the `metadata.finalizers` field. +- If a **validating webhook** prevents deletion (e.g., for the `system-project`), please refer to the [Bypassing the Webhook](../../../../reference-guides/rancher-webhook.md#bypassing-the-webhook) documentation. + ## Rolling Back If your upgrade does not complete successfully, you can roll back Rancher server and its data back to its last healthy state. For more information, see [Docker Rollback](roll-back-docker-installed-rancher.md). diff --git a/versioned_docs/version-2.11/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/versioned_docs/version-2.11/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md index 844fbaad30f..20d09a90ca0 100644 --- a/versioned_docs/version-2.11/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +++ b/versioned_docs/version-2.11/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -170,6 +170,13 @@ You can define the default behavior for newly created clusters or existing ones Changes to the global **imported-cluster-version-management** setting take effect during the cluster’s next reconciliation cycle. +:::note + +If version management is enabled for a cluster, Rancher will deploy the system-upgrade-controller app, along with the associated Plans and other required Kubernetes resources, to the cluster. +If version management is disabled, Rancher will remove these components from the cluster. + +::: + ## Configuring RKE2 and K3s Cluster Upgrades :::tip diff --git a/versioned_docs/version-2.11/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/versioned_docs/version-2.11/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md index 9d6fd0b1876..b5ce5400ff9 100644 --- a/versioned_docs/version-2.11/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md +++ b/versioned_docs/version-2.11/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -19,6 +19,7 @@ In order to deploy and run the adapter successfully, you need to ensure its vers | Rancher Version | Adapter Version | |-----------------|------------------| +| v2.11.4 | v106.0.0+up6.0.0 | | v2.11.3 | v106.0.0+up6.0.0 | | v2.11.2 | v106.0.0+up6.0.0 | | v2.11.1 | v106.0.0+up6.0.0 | diff --git a/versioned_docs/version-2.11/reference-guides/rancher-webhook.md b/versioned_docs/version-2.11/reference-guides/rancher-webhook.md index 760e51bd32d..fbb8d07514d 100644 --- a/versioned_docs/version-2.11/reference-guides/rancher-webhook.md +++ b/versioned_docs/version-2.11/reference-guides/rancher-webhook.md @@ -20,6 +20,7 @@ Each Rancher version is designed to be compatible with a single version of the w | Rancher Version | Webhook Version | Availability in Prime | Availability in Community | |-----------------|-----------------|-----------------------|---------------------------| +| v2.11.4 | v0.7.4 | ✓ | ✗ | | v2.11.3 | v0.7.3 | ✓ | ✓ | | v2.11.2 | v0.7.2 | ✓ | ✓ | | v2.11.1 | v0.7.1 | ✓ | ✓ | diff --git a/versioned_docs/version-2.12/api/api-reference.mdx b/versioned_docs/version-2.12/api/api-reference.mdx index c0eedae1159..ca78ba81cc8 100644 --- a/versioned_docs/version-2.12/api/api-reference.mdx +++ b/versioned_docs/version-2.12/api/api-reference.mdx @@ -15,4 +15,4 @@ At this time, not all Rancher resources are available through the Rancher Kubern import ApiDocMdx from '@theme/ApiDocMdx'; - + diff --git a/versioned_docs/version-2.12/api/extension-apiserver.md b/versioned_docs/version-2.12/api/extension-apiserver.md index b80de8af261..fa4556046b9 100644 --- a/versioned_docs/version-2.12/api/extension-apiserver.md +++ b/versioned_docs/version-2.12/api/extension-apiserver.md @@ -16,4 +16,4 @@ All versions of Kubernetes supported by Rancher with the feature will have the a :::note If the underlying Kubernetes distribution does not support the aggregation layer, you must migrate to a Kubernetes distribution that does before upgrading. -::: \ No newline at end of file +::: diff --git a/versioned_docs/version-2.12/api/v3-rancher-api-guide.md b/versioned_docs/version-2.12/api/v3-rancher-api-guide.md index 14bc92a5bed..e04b0896c0e 100644 --- a/versioned_docs/version-2.12/api/v3-rancher-api-guide.md +++ b/versioned_docs/version-2.12/api/v3-rancher-api-guide.md @@ -72,7 +72,7 @@ API responses are paginated with a limit of 100 resources per page by default. T ## Capturing v3 API Calls -You can use browser developer tools to capture how the v3 API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning an RKE cluster: +You can use browser developer tools to capture how the v3 API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning a Rancher Kubernetes distribution cluster: 1. In the Rancher UI, go to **Cluster Management** and click **Create.** 1. Click one of the cluster types. This example uses Digital Ocean. diff --git a/versioned_docs/version-2.12/api/workflows/kubeconfigs.md b/versioned_docs/version-2.12/api/workflows/kubeconfigs.md new file mode 100644 index 00000000000..a6e156f3272 --- /dev/null +++ b/versioned_docs/version-2.12/api/workflows/kubeconfigs.md @@ -0,0 +1,205 @@ +--- +title: Kubeconfigs +--- + + + + + +## Kubeconfig Resource + +Kubeconfig is a Rancher resource `kubeconfigs.ext.cattle.io` that allows generating `v1.Config` kubeconfig files for interacting with Rancher and clusters managed by Rancher. + +```sh +kubectl api-resources --api-group=ext.cattle.io +``` + +To get a description of the fields and structure of the Kubeconfig resource, run: + +```sh +kubectl explain kubeconfigs.ext.cattle.io +``` + +## Feature Flag + +The Kubeconfigs Public API is available since Rancher v2.12.0 and is enabled by default. It can be disabled by setting the `ext-kubeconfigs` feature flag to `false`. + +```sh +kubectl patch feature ext-kubeconfigs -p '{"spec":{"value":false}}' +``` + +## Creating a Kubeconfig + +Only a **valid and active** Rancher user can create a Kubeconfig. For example, trying to create a Kubeconfig using a `system:admin` service account will lead to an error: + +```bash +kubectl create -o jsonpath='{.status.value}' -f -< + + + +## Token Resource + +Rancher has an imperative API resource `tokens.ext.cattle.io` that allows you to generate tokens for authenticating with Rancher. + +```sh +kubectl api-resources --api-group=ext.cattle.io +``` + +To get a description of the fields and structure of the Token resource, run: + +```sh +kubectl explain tokens.ext.cattle.io +``` + +## Feature Flag + +The Tokens Public API is available for Rancher v2.12.0 and later, and is enabled by default. You can disable the Tokens Public API by setting the `ext-tokens` feature flag to `false` as shown in the example `kubectl` command below: + +```sh +kubectl patch feature ext-tokens -p '{"spec":{"value":false}}' +``` + +## Creating a Token + +:::caution +The Token value is only returned once in the `status.value` field. +::: + +Only a **valid and active** Rancher user can create a Token. Otherwise, you will get an error displayed (`Error from server (Forbidden)...`) when attempting to create a Token. + +```bash +kubectl create -o jsonpath='{.status.value}' -f -< - -![Weave Logo](/img/weave-logo.png) - -Weave enables networking and network policy in Kubernetes clusters across the cloud. Additionally, it support encrypting traffic between the peers. - -Kubernetes workers should open TCP port `6783` (control port), UDP port `6783` and UDP port `6784` (data ports). See the [port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. - -For more information, see the following pages: - -- [Weave Net Official Site](https://github.com/weaveworks/weave/blob/master/site/overview.md) - ### RKE2 Kubernetes clusters -Out-of-the-box, Rancher provides the following CNI network providers for RKE2 Kubernetes clusters: [Canal](#canal) (see above section), Calico, and Cilium. +Out-of-the-box, Rancher provides the following CNI network providers for RKE2 Kubernetes clusters: Calico, Canal, Cilium, and Flannel. You can choose your CNI network provider when you create new Kubernetes clusters from Rancher. @@ -131,6 +80,20 @@ For more information, see the following pages: - [Project Calico Official Site](https://www.projectcalico.org/) - [Project Calico GitHub Page](https://github.com/projectcalico/calico) +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal is a CNI network provider that gives you the best of Flannel and Calico. It allows users to easily deploy Calico and Flannel networking together as a unified networking solution, combining Calico’s network policy enforcement with the rich superset of Calico (unencapsulated) and/or Flannel (encapsulated) network connectivity options. + +In Rancher, Canal is the default CNI network provider combined with Flannel and VXLAN encapsulation. + +Kubernetes workers should open UDP port `8472` (VXLAN) and TCP port `9099` (health checks). If using Wireguard, you should open UDP ports `51820` and `51821`. For more details, refer to [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md). + +![](/img/canal-diagram.png) + +For more information, see the [Canal GitHub Page.](https://github.com/projectcalico/canal) + #### Cilium ![Cilium Logo](/img/cilium-logo.png) @@ -158,6 +121,23 @@ spec: - remote-node ``` +#### Flannel + +![Flannel Logo](/img/flannel-logo.png) + +Flannel is a simple and easy way to configure L3 network fabric designed for Kubernetes. Flannel runs a single binary agent named flanneld on each host, which is responsible for allocating a subnet lease to each host out of a larger, preconfigured address space. Flannel uses either the Kubernetes API or etcd directly to store the network configuration, the allocated subnets, and any auxiliary data (such as the host's public IP). Packets are forwarded using one of several backend mechanisms, with the default encapsulation being [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan). + +Encapsulated traffic is unencrypted by default. Flannel provides two solutions for encryption: + +* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec), which makes use of [strongSwan](https://www.strongswan.org/) to establish encrypted IPSec tunnels between Kubernetes workers. It is an experimental backend for encryption. +* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard), which is a more faster-performing alternative to strongSwan. + +Kubernetes workers should open UDP port `8472` (VXLAN). See [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +![Flannel Diagram](/img/flannel-diagram.png) + +For more information, see the [Flannel GitHub Page](https://github.com/flannel-io/flannel). + ## CNI Features by Provider The following table summarizes the different features available for each CNI network provider provided by Rancher. @@ -196,4 +176,4 @@ Canal is the default CNI network provider. We recommend it for most use cases. I ## How can I configure a CNI network provider? -Please see [Cluster Options](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) on how to configure a network provider for your cluster. For more advanced configuration options, please see how to configure your cluster using a [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) and the options for [Network Plug-ins](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/). +Please see [Cluster Options](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) on how to configure a network provider for your cluster. For more advanced configuration options, please see how to configure your cluster using a [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#cluster-config-file-reference). diff --git a/versioned_docs/version-2.12/faq/deprecated-features.md b/versioned_docs/version-2.12/faq/deprecated-features.md index 221a39ae343..617d6e5f6bc 100644 --- a/versioned_docs/version-2.12/faq/deprecated-features.md +++ b/versioned_docs/version-2.12/faq/deprecated-features.md @@ -16,9 +16,7 @@ Rancher will publish deprecated features as part of the [release notes](https:// | Patch Version | Release Date | |---------------|---------------| -| [2.11.2](https://github.com/rancher/rancher/releases/tag/v2.11.2) | May 22, 2025 | -| [2.11.1](https://github.com/rancher/rancher/releases/tag/v2.11.1) | Apr 24, 2025 | -| [2.11.0](https://github.com/rancher/rancher/releases/tag/v2.11.0) | Mar 31, 2025 | +| [2.12.0](https://github.com/rancher/rancher/releases/tag/v2.12.0) | July 30, 2025 | ## What can I expect when a feature is marked for deprecation? diff --git a/versioned_docs/version-2.12/faq/dockershim.md b/versioned_docs/version-2.12/faq/dockershim.md deleted file mode 100644 index 387b20b3eee..00000000000 --- a/versioned_docs/version-2.12/faq/dockershim.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Dockershim FAQ ---- - - - - - -The Dockershim is the CRI compliant layer between the Kubelet and the Docker daemon. As part of the Kubernetes 1.20 release, the [deprecation of the in-tree Dockershim was announced](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/). Removal is currently scheduled for Kubernetes 1.24. For more information on the deprecation and its timelines, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). - -RKE clusters, starting with Kubernetes 1.21, now support the external Dockershim to continue leveraging Docker as the CRI runtime. We now implement the upstream open source community Dockershim announced by [Mirantis and Docker](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) to ensure RKE clusters can continue to leverage Docker. - -To enable the external Dockershim, configure the following option. - -``` -enable_cri_dockerd: true -``` - -For users looking to use another container runtime, Rancher has the edge-focused K3s and datacenter-focused RKE2 Kubernetes distributions that use containerd as the default runtime. Imported RKE2 and K3s Kubernetes clusters can then be upgraded and managed through Rancher even after the removal of in-tree Dockershim in Kubernetes 1.24. - -## FAQ - -
- -Q: Do I have to upgrade Rancher to get Rancher’s support of the upstream Dockershim? - -The upstream support of Dockershim begins for RKE in Kubernetes 1.21. You will need to be on Rancher 2.6 or above to have support for RKE with Kubernetes 1.21. See our [support matrix](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/) for details. - -
- -Q: I am currently on RKE with Kubernetes 1.20. Do I need to upgrade to RKE with Kubernetes 1.21 sooner to avoid being out of support for Dockershim? - -A: The version of Dockershim in RKE with Kubernetes 1.20 will continue to work and is not scheduled for removal upstream until Kubernetes 1.24. It will only emit a warning of its future deprecation, which Rancher has mitigated in RKE with Kubernetes 1.21. You can plan your upgrade to Kubernetes 1.21 as you would normally, but should consider enabling the external Dockershim by Kubernetes 1.22. The external Dockershim will need to be enabled before upgrading to Kubernetes 1.24, at which point the existing implementation will be removed. - -For more information on the deprecation and its timeline, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). - -
- -Q: What are my other options if I don’t want to depend on the Dockershim? - -A: You can use a runtime like containerd with Kubernetes that does not require Dockershim support. RKE2 or K3s are two options for doing this. - -
- -Q: If I am already using RKE1 and want to switch to RKE2, what are my migration options? - -A: Rancher is exploring the possibility of an in-place upgrade path. Alternatively you can always migrate workloads from one cluster to another using kubectl. - -
diff --git a/versioned_docs/version-2.12/faq/install-and-configure-kubectl.md b/versioned_docs/version-2.12/faq/install-and-configure-kubectl.md index 868eb42caac..3022cae6816 100644 --- a/versioned_docs/version-2.12/faq/install-and-configure-kubectl.md +++ b/versioned_docs/version-2.12/faq/install-and-configure-kubectl.md @@ -14,17 +14,11 @@ See [kubectl Installation](https://kubernetes.io/docs/tasks/tools/install-kubect ## Configuration -When you create a Kubernetes cluster with RKE, RKE creates a `kube_config_cluster.yml` in the local directory that contains credentials to connect to your new cluster with tools like `kubectl` or `helm`. - -You can copy this file as `$HOME/.kube/config` or if you are working with multiple Kubernetes clusters, set the `KUBECONFIG` environmental variable to the path of `kube_config_cluster.yml`. - -``` -export KUBECONFIG=$(pwd)/kube_config_cluster.yml -``` +When you create a Kubernetes cluster with RKE2/K3s, the Kubeconfig file is stored at `/etc/rancher/rke2/rke2.yaml` or `/etc/rancher/k3s/k3s.yaml` depending on your chosen distribution. These files are used to configure access to the Kubernetes cluster. Test your connectivity with `kubectl` and see if you can get the list of nodes back. -``` +```shell kubectl get nodes NAME STATUS ROLES AGE VERSION 165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 diff --git a/versioned_docs/version-2.12/faq/rancher-is-no-longer-needed.md b/versioned_docs/version-2.12/faq/rancher-is-no-longer-needed.md index 1752bd29b37..e1d1f759206 100644 --- a/versioned_docs/version-2.12/faq/rancher-is-no-longer-needed.md +++ b/versioned_docs/version-2.12/faq/rancher-is-no-longer-needed.md @@ -17,9 +17,9 @@ If Rancher is ever deleted or unrecoverable, all workloads in the downstream Kub The capability to access a downstream cluster without Rancher depends on the type of cluster and the way that the cluster was created. To summarize: -- **Registered clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. +- **Registered/Imported clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. - **Hosted Kubernetes clusters:** If you created the cluster in a cloud-hosted Kubernetes provider such as EKS, GKE, or AKS, you can continue to manage the cluster using your provider's cloud credentials. -- **RKE clusters:** To access an [RKE cluster,](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. (The authorized cluster endpoint is enabled by default for RKE clusters.) With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. +- **Rancher provisioned clusters:** To access an [RKE2/K3s cluster](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. ## What if I don't want Rancher anymore? @@ -56,10 +56,10 @@ To detach the cluster, **Result:** The registered cluster is detached from Rancher and functions normally outside of Rancher. -## What if I don't want my RKE cluster or hosted Kubernetes cluster managed by Rancher? +## What if I don't want my hosted Kubernetes cluster managed by Rancher? At this time, there is no functionality to detach these clusters from Rancher. In this context, "detach" is defined as the ability to remove Rancher components from the cluster and manage access to the cluster independently of Rancher. The capability to manage these clusters without Rancher is being tracked in this [issue.](https://github.com/rancher/rancher/issues/25234) -For information about how to access clusters if the Rancher server is deleted, refer to [this section.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) +For information about how to access clusters if the Rancher server is deleted, refer to [this section.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) \ No newline at end of file diff --git a/versioned_docs/version-2.12/faq/technical-items.md b/versioned_docs/version-2.12/faq/technical-items.md index 985cf6a5dbf..cdf29707617 100644 --- a/versioned_docs/version-2.12/faq/technical-items.md +++ b/versioned_docs/version-2.12/faq/technical-items.md @@ -64,7 +64,7 @@ The Layer-4 Load Balancer is created as `type: LoadBalancer`. In Kubernetes, thi ## Where is the state of Rancher stored? - Docker Install: in the embedded etcd of the `rancher/rancher` container, located at `/var/lib/rancher`. -- Kubernetes install: in the etcd of the RKE cluster created to run Rancher. +- Kubernetes install: default location is in the `/var/lib/rancher/rke2` or `/var/lib/rancher/k3s` directories of the respective RKE2/K3s cluster created to run Rancher. ## How are the supported Docker versions determined? @@ -99,7 +99,7 @@ When the node is removed from the cluster, and the node is cleaned, you can add ## How can I add more arguments/binds/environment variables to Kubernetes components in a Rancher Launched Kubernetes cluster? -You can add more arguments/binds/environment variables via the [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) option in Cluster Options. For more information, see the [Extra Args, Extra Binds, and Extra Environment Variables](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/) in the RKE documentation or browse the [Example Cluster.ymls](https://rancher.com/docs/rke/latest/en/example-yamls/). +You can add more arguments/binds/environment variables via the respective [RKE2 Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md#cluster-configuration) or [K3s Config File](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md#cluster-configuration). ## How do I check if my certificate chain is valid? diff --git a/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/feature-flags.md index ce153bce599..e8ef27b5c19 100644 --- a/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/feature-flags.md +++ b/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -26,32 +26,54 @@ The following is a list of feature flags available in Rancher. If you've upgrade - `imperative-api-extension`: Enables Rancher's [extension API server](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) to register new APIs to Kubernetes. This flag is enabled by default. See the [Extension API Server](../../../api/extension-apiserver.md) page for more information. - `istio-virtual-service-ui`: Enables a [visual interface](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md) to create, read, update, and delete Istio virtual services and destination rules, which are Istio traffic management features. - `legacy`: Enables a set of features from 2.5.x and earlier, that are slowly being phased out in favor of newer implementations. These are a mix of deprecated features as well as features that will eventually be available to newer versions. This flag is disabled by default on new Rancher installations. If you're upgrading from a previous version of Rancher, this flag is enabled. -- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream RKE2/K3s clusters, currently limited to imported clusters and the local cluster, with plans to expand support to node-driver clusters. +- `managed-system-upgrade-controller`: Enables the installation of the system-upgrade-controller app in downstream imported RKE2/K3s clusters, as well as in the local cluster if it is an RKE2/K3s cluster. + +:::note Important: + +This `managed-system-upgrade-controller` flag is intended for **internal use only** and does not have an associated Feature CR. Use with caution. + +To control whether Rancher should manage the Kubernetes version of imported RKE2/K3s clusters, it is recommended to use the [imported-cluster-version-management](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters) feature that is available in Rancher v2.11.0 or newer. + +::: + +:::danger + +If the `managed-system-upgrade-controller` flag was **disabled** in Rancher v2.10.x, and any imported RKE2/K3s clusters were upgraded **outside of Rancher**, follow the steps below to prevent the unexpected installation of the system-upgrade-controller app and to ensure the [imported-cluster-version-management](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters) feature works correctly: + +1. Upgrade Rancher to v2.11.0 or newer, making sure to **retain** the `managed-system-upgrade-controller=false` feature flag in Helm values if it was set during the v2.10.x installation. +1. After Rancher is fully up and running, disable the `imported-cluster-version-management` setting. You can do this either through the Rancher UI by clicking **☰ > Global Settings > Settings > imported-cluster-version-management**, or by editing the corresponding `Setting.management.cattle.io/v3` custom resource via kubectl. +1. Perform a second Helm upgrade, this time omitting the `managed-system-upgrade-controller=false` feature flag. + +Now, the imported cluster version management is disabled by default, and Rancher no longer installs the system-upgrade-controller app on imported clusters automatically. + +You can enable this feature on a per-cluster basis. For more information, please refer to the [documentation](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#configuring-version-management-for-rke2-and-k3s-clusters). + +::: + - `multi-cluster-management`: Allows multi-cluster provisioning and management of Kubernetes clusters. This flag can only be set at install time. It can't be enabled or disabled later. - `rke1-custom-node-cleanup`: Enables cleanup of deleted RKE1 custom nodes. We recommend that you keep this flag enabled, to prevent removed nodes from attempting to rejoin the cluster. - `rke2`: Enables provisioning RKE2 clusters. This flag is enabled by default. - `token-hashing`: Enables token hashing. Once enabled, existing tokens will be hashed and all new tokens will be hashed automatically with the SHA256 algorithm. Once a token is hashed it can't be undone. This flag can't be disabled after its enabled. See [API Tokens](../../../api/api-tokens.md#token-hashing) for more information. -- `uiextension`: Enables UI extensions. This flag is enabled by default. Enabling or disabling the flag forces the Rancher pod to restart. The first time this flag is set to `true`, it creates a CRD and enables the controllers and endpoints necessary for the feature to work. If set to `false`, it disables the previously mentioned controllers and endpoints. Setting `uiextension` to `false` has no effect on the CRD -- it does not create a CRD if it does not yet exist, nor does it delete the CRD if it already exists. +- `uiextension`: Enables UI extensions. This flag is enabled by default. Enabling or disabling the flag forces the Rancher pod to restart. The first time this flag is set to `Active`, it creates a CRD and enables the controllers and endpoints necessary for the feature to work. If set to `Disabled`, it disables the previously mentioned controllers and endpoints. Setting `uiextension` to `Disabled` has no effect on the CRD -- it does not create a CRD if it does not yet exist, nor does it delete the CRD if it already exists. - `unsupported-storage-drivers`: Enables types for storage providers and provisioners that aren't enabled by default. See [Allow Unsupported Storage Drivers](../../../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) for more information. -- `ui-sql-cache`: Enables a SQLite-based cache for UI tables. See [UI Server-Side Pagination](../../../how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md) for more information. - +- `ui-sql-cache`: Enables an SQLite-based cache for UI tables and Server-Side Pagination. See [UI Server-Side Pagination](../../../how-to-guides/advanced-user-guides/ui-server-side-pagination.md) for more information. The following table shows the availability and default values for some feature flags in Rancher. Features marked "GA" are generally available: | Feature Flag Name | Default Value | Status | Available As Of | Additional Information | | ----------------------------- | ------------- | ------------ | --------------- | ---------------------- | -| `aggregated-roletemplates` | `false` | Highly experimentatl | v2.11.0 | This flag value is locked on install and can't be changed. | -| `clean-stale-secrets` | `true` | GA | v2.10.2 | | -| `continuous-delivery` | `true` | GA | v2.6.0 | | -| `external-rules` | v2.7.14: `false`, v2.8.5: `true` | Removed | v2.7.14, v2.8.5 | This flag affected [external `RoleTemplate` behavior](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#external-roletemplate-behavior). It is removed in Rancher v2.9.0 and later as the behavior is enabled by default. | -| `fleet` | `true` | Can no longer be disabled | v2.6.0 | | -| `fleet` | `true` | GA | v2.5.0 | | -| `harvester` | `true` | Experimental | v2.6.1 | | -| `imperative-api-extension` | `true` | GA | v2.11.0 | | -| `legacy` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | | -| `managed-system-upgrade-controller` | `true` | GA | v2.10.0 | | +| `aggregated-roletemplates` | `Disabled` | Highly experimental | v2.11.0 | This flag value is locked on install and can't be changed. | +| `clean-stale-secrets` | `Active` | GA | v2.10.2 | | +| `continuous-delivery` | `Active` | GA | v2.6.0 | | +| `external-rules` | v2.7.14: `Disabled`, v2.8.5: `Active` | Removed | v2.7.14, v2.8.5 | This flag affected [external `RoleTemplate` behavior](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#external-roletemplate-behavior). It is removed in Rancher v2.9.0 and later as the behavior is enabled by default. | +| `fleet` | `Active` | Can no longer be disabled | v2.6.0 | | +| `fleet` | `Active` | GA | v2.5.0 | | +| `harvester` | `Active` | Experimental | v2.6.1 | | +| `imperative-api-extension` | `Active` | GA | v2.11.0 | | +| `legacy` | `Disabled` for new installs, `Active` for upgrades | GA | v2.6.0 | | +| `managed-system-upgrade-controller` | `Active` | GA | v2.10.0 | | | `rke1-custom-node-cleanup`| `true` | GA | v2.6.0 | | | `rke2` | `true` | Experimental | v2.6.0 | | -| `token-hashing` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | | -| `uiextension` | `true` | GA | v2.9.0 | | -| `ui-sql-cache` | `false` | Highly experimental | v2.9.0 | | +| `token-hashing` | `Disabled` for new installs, `Active` for upgrades | GA | v2.6.0 | | +| `uiextension` | `Active` | GA | v2.9.0 | | +| `ui-sql-cache` | `Active` | GA | v2.9.0 | | diff --git a/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md b/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md index aa8da6f793f..8dedc6288bf 100644 --- a/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md +++ b/versioned_docs/version-2.12/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md @@ -36,7 +36,8 @@ For information on enabling experimental features, refer to [this page.](../../. | `antiAffinity` | "preferred" | `string` - AntiAffinity rule for Rancher pods - "preferred, required" | | `auditLog.destination` | "sidecar" | `string` - Stream to sidecar container console or hostPath volume - "sidecar, hostPath" | | `auditLog.hostPath` | "/var/log/rancher/audit" | `string` - log file destination on host (only applies when `auditLog.destination` is set to `hostPath`) | -| `auditLog.level` | 0 | `int` - set the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) level. 0 is off. [0-3] | +| `auditLog.enabled` | false | `bool` - Enables / disables audit logging. | +| `auditLog.level` | 0 | `int` - Sets the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) level [0-3]. | | `auditLog.maxAge` | 1 | `int` - maximum number of days to retain old audit log files (only applies when `auditLog.destination` is set to `hostPath`) | | `auditLog.maxBackup` | 1 | `int` - maximum number of audit log files to retain (only applies when `auditLog.destination` is set to `hostPath`) | | `auditLog.maxSize` | 100 | `int` - maximum size in megabytes of the audit log file before it gets rotated (only applies when `auditLog.destination` is set to `hostPath`) | @@ -63,6 +64,10 @@ For information on enabling experimental features, refer to [this page.](../../. | `tls` | "ingress" | `string` - See [External TLS Termination](#external-tls-termination) for details. - "ingress, external" | | `useBundledSystemChart` | `false` | `bool` - select to use the system-charts packaged with Rancher server. This option is used for air gapped installations. | + +When using Rancher v2.12.0 and above, Rancher will use an audit logging controller that watches `AuditPolicy` CRs for configuring additional redactions, for more info see [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md). + + ### Bootstrap Password You can [set a specific bootstrap password](../resources/bootstrap-password.md) during Rancher installation. If you don't set a specific bootstrap password, Rancher randomly generates a password for the first admin account. diff --git a/versioned_docs/version-2.12/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/versioned_docs/version-2.12/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md index c4618be475f..fbdc5c0356e 100644 --- a/versioned_docs/version-2.12/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md +++ b/versioned_docs/version-2.12/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -53,6 +53,10 @@ You can obtain `` and `` by loggi ## Upgrade +:::danger +Rancher upgrades to version 2.12.0 and later will be blocked if any RKE1-related resources are detected, as the Rancher Kubernetes Engine (RKE/RKE1) is end of life as of **July 31, 2025**. For detailed cleanup and recovery steps, refer to the [RKE1 Resource Validation and Upgrade Requirements in Rancher v2.12](#rke1-resource-validation-and-upgrade-requirements-in-rancher-v212). +::: + During upgrade, you create a copy of the data from your current Rancher container and a backup in case something goes wrong. Then you deploy the new version of Rancher in a new container using your existing data. ### 1. Create a copy of the data from your Rancher server container @@ -388,6 +392,45 @@ See [Restoring Cluster Networking](https://github.com/rancher/rancher-docs/tree/ Remove the previous Rancher server container. If you only stop the previous Rancher server container (and don't remove it), the container may restart after the next server reboot. +## RKE1 Resource Validation and Upgrade Requirements in Rancher v2.12 + +Rancher v2.12.0 and later has removed support for the Rancher Kubernetes Engine (RKE/RKE1). During upgrade, Rancher validates the cluster resources and blocks the upgrade if any RKE1-related resources are detected. + +This validation affects the following resource types: + +- Clusters with `rkeConfig` (`clusters.management.cattle.io`) +- NodeTemplates (`nodetemplates.management.cattle.io`) +- ClusterTemplates (`clustertemplates.management.cattle.io`) + +This is particularly relevant for single-node Docker installations, where Rancher is not running during the upgrade. In such cases, controllers are not available to automatically clean up deprecated resources, and the upgrade process will fail early with an error listing the blocking resources. + +### 1. Pre-Upgrade (Recommended) + +Before upgrading, while Rancher is still running: + +- Run the `pre-upgrade-hook` cleanup script to delete all RKE1 clusters and templates. You can find the script in the Rancher GitHub repository: [pre-upgrade-hook.sh](https://github.com/rancher/rancher/blob/v2.12.0/chart/scripts/pre-upgrade-hook.sh). +- This allows Rancher to clean up associated resources and finalizers. + +### 2. Post-Upgrade Failure Due to Residual RKE1 Resources + +If the upgrade to Rancher v2.12.0 or later is attempted without prior cleanup of RKE1 resources: + +- The upgrade will fail and display an error listing the resource names that are preventing the upgrade. +- This occurs because Rancher includes validation to detect and block upgrades when unsupported RKE1 resources are still present. +- To proceed, [rollback](#rolling-back) to the previous Rancher version, delete the identified resources, and then retry after [manual cleanup](#manual-cleanup-after-rollback). + +:::note Helm-based Rancher +Helm-based Rancher installations are not affected by this issue, as Rancher remains available during the upgrade and can perform resource cleanup as needed. +::: + +### Manual Cleanup After Rollback + +Users should perform the following steps after rolling back to a previous Rancher version: + +- **Manually delete** the resources listed in the upgrade error message (e.g., RKE1 clusters, NodeTemplates, ClusterTemplates). +- If deletion is blocked due to **finalizers**, edit the resources and remove the `metadata.finalizers` field. +- If a **validating webhook** prevents deletion (e.g., for the `system-project`), please refer to the [Bypassing the Webhook](../../../../reference-guides/rancher-webhook.md#bypassing-the-webhook) documentation. + ## Rolling Back If your upgrade does not complete successfully, you can roll back Rancher server and its data back to its last healthy state. For more information, see [Docker Rollback](roll-back-docker-installed-rancher.md). diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md deleted file mode 100644 index a7c6ed43472..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: CIS Scan Guides ---- - - - - - -- [Install rancher-cis-benchmark](install-rancher-cis-benchmark.md) -- [Uninstall rancher-cis-benchmark](uninstall-rancher-cis-benchmark.md) -- [Run a Scan](run-a-scan.md) -- [Run a Scan Periodically on a Schedule](run-a-scan-periodically-on-a-schedule.md) -- [Skip Tests](skip-tests.md) -- [View Reports](view-reports.md) -- [Enable Alerting for rancher-cis-benchmark](enable-alerting-for-rancher-cis-benchmark.md) -- [Configure Alerts for Periodic Scan on a Schedule](configure-alerts-for-periodic-scan-on-a-schedule.md) -- [Create a Custom Benchmark Version to Run](create-a-custom-benchmark-version-to-run.md) \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md deleted file mode 100644 index 8d3b66c7e4e..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Create a Custom Benchmark Version for Running a Cluster Scan ---- - - - - - -There could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. - -It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. - -For details, see [this page.](../../../integrations-in-rancher/cis-scans/custom-benchmark.md) \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md deleted file mode 100644 index ef2b5ae330d..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Enable Alerting for Rancher CIS Benchmark ---- - - - - - -Alerts can be configured to be sent out for a scan that runs on a schedule. - -:::note Prerequisite: - -Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) - -While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) - -::: - -While installing or upgrading the `rancher-cis-benchmark` Helm chart, set the following flag to `true` in the `values.yaml`: - -```yaml -alerts: - enabled: true -``` \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md deleted file mode 100644 index 2fede69bee6..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Run a Scan ---- - - - - - -When a ClusterScan custom resource is created, it launches a new CIS scan on the cluster for the chosen ClusterScanProfile. - -:::note - -There is currently a limitation of running only one CIS scan at a time for a cluster. If you create multiple ClusterScan custom resources, they will be run one after the other by the operator, and until one scan finishes, the rest of the ClusterScan custom resources will be in the "Pending" state. - -::: - -To run a scan, - -1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. -1. Click **Create**. -1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. -1. Click **Create**. - -**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md deleted file mode 100644 index 7492bc03f0b..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Skip Tests ---- - - - - - -CIS scans can be run using test profiles with user-defined skips. - -To skip tests, you will create a custom CIS scan profile. A profile contains the configuration for the CIS scan, which includes the benchmark versions to use and any specific tests to skip in that benchmark. - -1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Profile**. -1. From here, you can create a profile in multiple ways. To make a new profile, click **Create** and fill out the form in the UI. To make a new profile based on an existing profile, go to the existing profile and click **⋮ Clone**. If you are filling out the form, add the tests to skip using the test IDs, using the relevant CIS Benchmark as a reference. If you are creating the new test profile as YAML, you will add the IDs of the tests to skip in the `skipTests` directive. You will also give the profile a name: - - ```yaml - apiVersion: cis.cattle.io/v1 - kind: ClusterScanProfile - metadata: - annotations: - meta.helm.sh/release-name: clusterscan-operator - meta.helm.sh/release-namespace: cis-operator-system - labels: - app.kubernetes.io/managed-by: Helm - name: "" - spec: - benchmarkVersion: cis-1.5 - skipTests: - - "1.1.20" - - "1.1.21" - ``` -1. Click **Create**. - -**Result:** A new CIS scan profile is created. - -When you [run a scan](./run-a-scan.md) that uses this profile, the defined tests will be skipped during the scan. The skipped tests will be marked in the generated report as `Skip`. diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md deleted file mode 100644 index df23f7abbdc..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Uninstall Rancher CIS Benchmark ---- - - - - - -1. From the **Cluster Dashboard,** go to the left navigation bar and click **Apps > Installed Apps**. -1. Go to the `cis-operator-system` namespace and check the boxes next to `rancher-cis-benchmark-crd` and `rancher-cis-benchmark`. -1. Click **Delete** and confirm **Delete**. - -**Result:** The `rancher-cis-benchmark` application is uninstalled. \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md deleted file mode 100644 index bb9045033bc..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: View Reports ---- - - - - - -To view the generated CIS scan reports, - -1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. -1. The **Scans** page will show the generated reports. To see a detailed report, go to a scan report and click the name. - -One can download the report from the Scans list or from the scan detail page. - -To get the verbose version of the CIS scan results, run the following command on the cluster that was scanned. Note that the scan must be completed before this can be done. - -```console -export REPORT="scan-report-name" -kubectl get clusterscanreport $REPORT -o json |jq ".spec.reportJSON | fromjson" | jq -r ".actual_value_map_data" | base64 -d | gunzip | jq . -``` diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md new file mode 100644 index 00000000000..c90922ec778 --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md @@ -0,0 +1,16 @@ +--- +title: Compliance Scan Guides +--- + + + + + +- [Install rancher-compliance](install-rancher-compliance.md) +- [Uninstall rancher-compliance](uninstall-rancher-compliance.md) +- [Run a Scan](run-a-scan.md) +- [Run a Scan Periodically on a Schedule](run-a-scan-periodically-on-a-schedule.md) +- [View Reports](view-reports.md) +- [Enable Alerting for rancher-compliance](enable-alerting-for-rancher-compliance.md) +- [Configure Alerts for Periodic Scan on a Schedule](configure-alerts-for-periodic-scan-on-a-schedule.md) +- [Create a Custom Benchmark Version to Run](create-a-custom-compliance-version-to-run.md) diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md similarity index 60% rename from docs/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md rename to versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md index 204f95c05bd..3f53f30dc9a 100644 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md @@ -3,7 +3,7 @@ title: Configure Alerts for Periodic Scan on a Schedule --- - + It is possible to run a ClusterScan on a schedule. @@ -12,27 +12,27 @@ A scheduled scan can also specify if you should receive alerts when the scan com Alerts are supported only for a scan that runs on a schedule. -The CIS Benchmark application supports two types of alerts: +The compliance application supports two types of alerts: - Alert on scan completion: This alert is sent out when the scan run finishes. The alert includes details including the ClusterScan's name and the ClusterScanProfile name. - Alert on scan failure: This alert is sent out if there are some test failures in the scan run or if the scan is in a `Fail` state. :::note Prerequisite -Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) +Before enabling alerts for `rancher-compliance`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) -While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) +While configuring the routes for `rancher-compliance` alerts, you can specify the matching using the key-value pair `job: rancher-compliance-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-compliance-scan-alerts) ::: To configure alerts for a scan that runs on a schedule, -1. Please enable alerts on the `rancher-cis-benchmark` application. For more information, see [this page](../../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md). +1. Please enable alerts on the `rancher-compliance` application. For more information, see [this page](../../../how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md). 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. +1. On the **Clusters** page, go to the cluster where you want to run a Compliance scan and click **Explore**. +1. Click **compliance > Scan**. 1. Click **Create**. -1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the Compliance Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. 1. Choose the option **Run scan on a schedule**. 1. Enter a valid [cron schedule expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) in the field **Schedule**. 1. Check the boxes next to the Alert types under **Alerting**. @@ -41,4 +41,4 @@ To configure alerts for a scan that runs on a schedule, **Result:** The scan runs and reschedules to run according to the cron schedule provided. Alerts are sent out when the scan finishes if routes and receiver are configured under `rancher-monitoring` application. -A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. \ No newline at end of file +A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run.md new file mode 100644 index 00000000000..a15fc96a7fd --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run.md @@ -0,0 +1,13 @@ +--- +title: Create a Custom Compliance Version for Running a Cluster Scan +--- + + + + + +There could be some Kubernetes cluster setups that require custom configurations of the Compliance tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream Compliance look for them. + +It is now possible to create a custom compliance version for running a cluster scan using the `rancher-compliance` application. + +For details, see [this page.](../../../integrations-in-rancher/compliance-scans/custom-benchmark.md) \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md new file mode 100644 index 00000000000..d5328a0dd0c --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md @@ -0,0 +1,24 @@ +--- +title: Enable Alerting for Rancher Compliance +--- + + + + + +Alerts can be configured to be sent out for a scan that runs on a schedule. + +:::note Prerequisite: + +Before enabling alerts for `rancher-compliance`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) + +While configuring the routes for `rancher-compliance` alerts, you can specify the matching using the key-value pair `job: rancher-compliance-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-compliance-scan-alerts) + +::: + +While installing or upgrading the `rancher-compliance` Helm chart, set the following flag to `true` in the `values.yaml`: + +```yaml +alerts: + enabled: true +``` \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance.md similarity index 51% rename from versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md rename to versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance.md index c6987a97c64..c00eab90648 100644 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance.md @@ -1,15 +1,15 @@ --- -title: Install Rancher CIS Benchmark +title: Install Rancher Compliance --- - + 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to install CIS Benchmark and click **Explore**. +1. On the **Clusters** page, go to the cluster where you want to install Compliance and click **Explore**. 1. In the left navigation bar, click **Apps > Charts**. -1. Click **CIS Benchmark** +1. Click **Compliance** 1. Click **Install**. -**Result:** The CIS scan application is deployed on the Kubernetes cluster. +**Result:** The compliance scan application is deployed on the Kubernetes cluster. diff --git a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule.md similarity index 75% rename from docs/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md rename to versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule.md index 076fbdf409b..ed0d251592e 100644 --- a/docs/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule.md @@ -3,15 +3,15 @@ title: Run a Scan Periodically on a Schedule --- - + To run a ClusterScan on a schedule, 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. -1. Click **CIS Benchmark > Scan**. -1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. On the **Clusters** page, go to the cluster where you want to run a Compliance scan and click **Explore**. +1. Click **Compliance > Scan**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the Compliance Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. 1. Choose the option **Run scan on a schedule**. 1. Enter a valid cron schedule expression in the field **Schedule**. 1. Choose a **Retention** count, which indicates the number of reports maintained for this recurring scan. By default this count is 3. When this retention limit is reached, older reports will get purged. @@ -21,4 +21,4 @@ To run a ClusterScan on a schedule, A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. -You can also see the previous reports by choosing the report from the **Reports** dropdown on the scan detail page. \ No newline at end of file +You can also see the previous reports by choosing the report from the **Reports** dropdown on the scan detail page. diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan.md new file mode 100644 index 00000000000..5b91cdb7e52 --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan.md @@ -0,0 +1,26 @@ +--- +title: Run a Scan +--- + + + + + +When a ClusterScan custom resource is created, it launches a new compliance scan on the cluster for the chosen ClusterScanProfile. + +:::note + +There is currently a limitation of running only one compliance scan at a time for a cluster. If you create multiple ClusterScan custom resources, they will be run one after the other by the operator, and until one scan finishes, the rest of the ClusterScan custom resources will be in the "Pending" state. + +::: + +To run a scan, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a compliance scan and click **Explore**. +1. Click **Compliance > Scan**. +1. Click **Create**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the Compliance Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Click **Create**. + +**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance.md new file mode 100644 index 00000000000..313acf79555 --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance.md @@ -0,0 +1,13 @@ +--- +title: Uninstall Rancher Compliance +--- + + + + + +1. From the **Cluster Dashboard,** go to the left navigation bar and click **Apps > Installed Apps**. +1. Go to the `compliance-operator-system` namespace and check the boxes next to `rancher-compliance-crd` and `rancher-compliance`. +1. Click **Delete** and confirm **Delete**. + +**Result:** The `rancher-compliance` application is uninstalled. \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports.md new file mode 100644 index 00000000000..ad042390485 --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports.md @@ -0,0 +1,23 @@ +--- +title: View Reports +--- + + + + + +To view the generated Compliance scan reports, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a Compliance scan and click **Explore**. +1. Click **Compliance > Scan**. +1. The **Scans** page will show the generated reports. To see a detailed report, go to a scan report and click the name. + +One can download the report from the Scans list or from the scan detail page. + +To get the verbose version of the compliance scan results, run the following command on the cluster that was scanned. Note that the scan must be completed before this can be done. + +```console +export REPORT="scan-report-name" +kubectl get clusterscanreports.compliance.cattle.io $REPORT -o json |jq ".spec.reportJSON | fromjson" | jq -r ".actual_value_map_data" | base64 -d | gunzip | jq . +``` diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md index f7266be3ff3..7a4384b4dad 100644 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md @@ -185,6 +185,7 @@ For help troubleshooting certificates, see [this section.](../../getting-started If you want to record all transactions with the Rancher API, enable the [API Auditing](enable-api-audit-log.md) feature by adding the flags below into your install command. -e AUDIT_LEVEL=1 \ + -e AUDIT_LOG_ENABLED=true \ -e AUDIT_LOG_PATH=/var/log/auditlog/rancher-api-audit.log \ -e AUDIT_LOG_MAXAGE=20 \ -e AUDIT_LOG_MAXBACKUP=20 \ diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-oidc-provider.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-oidc-provider.md new file mode 100644 index 00000000000..9f85fd52168 --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/configure-oidc-provider.md @@ -0,0 +1,138 @@ +--- +title: Configure Rancher as an OIDC provider +--- + + + + + +Rancher can function as a standard OpenID Connect (OIDC) provider, allowing external applications to use Rancher for authentication. +This can be used for enabling single sign-on (SSO) across Rancher Prime components. For example, see the [documentation](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/security/authentication/oidc.html) for configuring the OIDC provider for SUSE Observability. + +The OIDC provider can be enabled with the `oidc-provider` feature flag. When this flag is on the following endpoints are available: + +- `https://{rancher-url}/oidc/authorize`: This endpoint initiates the authentication flow. If a user is already logged into Rancher, it returns an authorization code. Otherwise, it redirects the user to the Rancher login page. Authorization codes and related request information are securely stored in session secrets. Codes are single-use and expire after 10 minutes. + +- `https://{rancher-url}/oidc/token`: This endpoint exchanges an authorization code for an `id_token`, `access_token`, and `refresh_token`. + +- `https://{rancher-url}/oidc/.well-known/openid-configuration`: This endpoint returns a JSON document containing the OIDC provider's configuration, including endpoint URLs, supported scopes, claims, and other relevant details. + +- `https://{rancher-url}/oidc/userinfo`: This endpoint provides information about the authenticated user. + +The OIDC provider supports the OIDC Authentication Code Flow with PKCE. + +## Configure OIDCClient + +An `OIDCClient` represents an external application that will be authenticating against Rancher. + +### Programmatically + +Create an `OIDCClient`: + +```yaml +apiVersion: management.cattle.io/v3 +kind: OIDCClient +metadata: + name: oidc-client-test +spec: + tokenExpirationSeconds: 600 # expiration of the id_token and access_token + refreshTokenExpirationSeconds: 3600 # expiration of the refresh_token + redirectURIs: + - "https://myredirecturl.com" # replace with your redirect url +``` +Rancher automatically generates a client ID and client secret for each `OIDCClient`. +Once the resource is created, Rancher populates the status field with the client id: + +```yaml +apiVersion: management.cattle.io/v3 +kind: OIDCClient +metadata: + name: oidc-client-test +spec: + tokenExpirationSeconds: 600 # expiration of the id_token and access_token + refreshTokenExpirationSeconds: 3600 # expiration of the refresh_token + redirectURIs: + - "https://myredirecturl.com" # replace with your redirect url +status: + clientID: client-xxx + clientSecrets: + client-secret-1: + createdAt: "xxx" + lastFiveCharacters: xxx +``` + +Rancher automatically generates a Kubernetes `Secret` in the `cattle-oidc-client-secrets` namespace for each `OIDCClient` resource. The Secret's name matches the `OIDCClient` client ID. +Initially, the `Secret` contains a single client secret. + +To retrieve the client secret: + +``` +kubectl get secret client-xxx -n cattle-oidc-client-secrets -o jsonpath="{.data.client-secret-1}" | base64 -d +``` + +Output: + +``` +secret-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +You can now use this client ID and client secret in your OIDC client application. + +#### Managing Client Secrets + +You can manage multiple client secrets per `OIDCClient`. Use annotations on the `OIDCClient` resource to perform secret operations: + +- Creation: Adding the `cattle.io/oidc-client-secret-create: true` annotation triggers the creation of a new client secret. +- Removal: Adding the `cattle.io/oidc-client-secret-remove:client-secret-1` annotation removes the specified client secrets. +- Regeneration: Adding the `cattle.io/oidc-client-secret-regenerate:client-secret-1` annotation regenerates the specified client secrets. + +### Rancher UI + +Create an OIDCClient: + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **OIDC Apps**. +1. Click **Add Application**. Fill out the **Create OIDC App** form. +1. Click **Add Application**. + +#### Managing Client Secrets + +In the OIDC App page: + +- Creation: Click **Add new secret**. +- Removal: Click **⋮ > Delete** +- Regeneration: Click **⋮ > Regenerate** + +## Signing key + +A default key pair for signing the `id_token`, `access_token`, and `refresh_token` tokens is created by Rancher in a `Secret` called `oidc-signing-key` in the `cattle-system` namespace. Only one key will be used for signing, but multiple public keys can be returned in the jwks endpoint in order to avoid disruption when doing a key rotation. + +### Rotation without disruption + +In order to create a new key pair for signing you need to manually create a new keypair and add it to the `oidc-signing-key` `Secret` + +Example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: oidc-signing-key +type: Opaque +data: + key2.pem: + key1.pub: + key2.pub: +``` + +Rancher will sign tokens using `key2.pem`, while the JWKS endpoint will serve both `key1.pub` and `key2.pub`. This ensures a smooth +key rotation from `key1` to `key2` without disrupting existing token verification. Note that only one private key (.pem) can be stored in the +secret at a time, and each key pair must share the same base name, differing only by their suffix: .pem for the private key and .pub for the public key. + +### Rotation with disruption + +Removing the `oidc-signing-key` `Secret` will cause Rancher to regenerate the signing key on the next restart. + +:::warning +This will invalidate all previously issued `id_token`, `access_token`, and `refresh_token` tokens making them unusable. +::: diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-api-audit-log.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-api-audit-log.md index 4f0a15c7984..36e9a7f5ad3 100644 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-api-audit-log.md +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-api-audit-log.md @@ -24,7 +24,8 @@ The usage below defines rules about what the audit log should record and what da | Parameter | Description | | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `AUDIT_LEVEL` | `0` - Disable audit log (default setting).
`1` - Log event metadata.
`2` - Log event metadata and request body.
`3` - Log event metadata, request body, and response body. Each log transaction for a request/response pair uses the same `auditID` value.

See [Audit Level Logging](#audit-log-levels) for a table that displays what each setting logs. | +| `AUDIT_LOG_ENABLED` | `false` - Disables the audit log (default setting).
`true` - Enables the audit log. | +| `AUDIT_LEVEL` | `0` - Log request and response metadata (default setting).
`1` - Log request and response headers.
`2` - Log request body.
`3` - Log response body. Each log level is cumulative and each subsequent level logs the previous level data. Each log transaction for a request/response pair uses the same `auditID` value.

See [Audit Level Logging](#audit-log-levels) for a table that displays what each setting logs. | | `AUDIT_LOG_PATH` | Log path for Rancher Server API. Default path is `/var/log/auditlog/rancher-api-audit.log`. You can mount the log directory to host.

Usage Example: `AUDIT_LOG_PATH=/my/custom/path/`
| | `AUDIT_LOG_MAXAGE` | Defined the maximum number of days to retain old audit log files. Default is 10 days. | | `AUDIT_LOG_MAXBACKUP` | Defines the maximum number of audit log files to retain. Default is 10. | @@ -36,12 +37,138 @@ The usage below defines rules about what the audit log should record and what da The following table displays what parts of API transactions are logged for each [`AUDIT_LEVEL`](#api-audit-log-options) setting. -| `AUDIT_LEVEL` Setting | Metadata | Request Body | Response Body | -| --------------------- | -------- | ------------ | ------------- | -| `0` | | | | -| `1` | ✓ | | | -| `2` | ✓ | ✓ | | -| `3` | ✓ | ✓ | ✓ | + +| `AUDIT_LEVEL` Setting | Metadata | Request Headers | Response Headers | Request Body | Response Body | +|--------------------------------|----------------|---------------------------|-----------------------------|----------------------|-----------------------| +| 0 | ✓ | +| 1 | ✓ | ✓ | ✓ +| 2 | ✓ | ✓ | ✓ | ✓ +| 3 | ✓ | ✓ | ✓ | ✓ | ✓ + +## Audit Log policies + +Audit log policies allow end users to configure redactions using `AuditPolicy` cluster-scoped CRs in addition to the [default redactions and filters](#default-redactions--filters). + +All configured audit log policies are additive. + +Redaction policies for headers utilize a regular expression (regex) engine to redact headers, while a JSONPath engine is used to redact request/response headers. + +The JSONPath engine does not support the script or filter expressions. For getting started with JSONPath expressions, a good resource to consult is [Stafan Goessner's article on JSONPath](https://goessner.net/articles/JsonPath/). + +The structure of an audit policy CR is as follows: + +```yaml +apiVersion: auditlog.cattle.io/v1 +kind: AuditPolicy +spec: + enabled : true # true/false + # list of API request filters + filters: + - action: allow # allow/deny + # would allow logs sent to "/foo/some/endpoint" but not "/foo" or "/foobar". + requestURI: "/foo/.*" + # additionalRedactions allows configuration of redactions on headers using `jsonpath` expressions + additionalRedactions: + # redacts headers based on regex expressions + - headers: + - "Cache.*" + # paths redacts information from request and response bodies based on json path expressions + paths: + - "$.gitCommit" + verbosity: + level : 0 # matches the levels in the above audit log table + # request allows fine-grained control over which request data + # gets included. This overrides the behaviour of the generic verbosity.level + request: + headers : true # true/false + body : true # true/false + # response allows fine-grained control over which response data + # gets included. This overrides the behaviour of the generic verbosity.level + response: + headers : true # true/false + body: true # true/false +``` + +### Examples + +The following example shows only logging requests containing `login` in the request path to the audit log path: + +```yaml +apiVersion: auditlog.cattle.io/v1 +kind: AuditPolicy +metadata: + name: filters +spec: + filters: + - action: deny + requestUri: ".*" + - action: allow + requestUri: ".*login.*" +``` + +The following example shows how to redact specific fields containing `gitCommint` in request/response bodies: + +```yaml +apiVersion: auditlog.cattle.io/v1 +kind: AuditPolicy +metadata: + name: redactions +spec: + additionalRedactions: + - paths: + - "$.gitCommit" +``` + +### Default redactions & filters + +The audit log controller comes with default built-in redactions for common sensitive information. + +#### Redacted headers + +Generic headers: +- `Cookie` +- `Set-Cookie` +- `X-Api-Set-Cookie-Header` +- `Authorization` +- `X-Api-Tunnel-Params` +- `X-Api-Tunnel-Token` +- `X-Api-Auth-Header` +- `X-Amz-Security-Token` + + +#### Redacted body fields + +Generic body fields: + +- `credentials` +- `applicationSecret` +- `oauthCredential` +- `serviceAccountCredential` +- `spKey` +- `spCert` +- `certificate` +- `privateKey` +- `secretsEncryptionConfig` +- `manifestUrl` +- `insecureWindowsNodeCommand` +- `insecureNodeCommand` +- `insecureCommand` +- `command` +- `nodeCommand` +- `windowsNodeCommand` +- `clientRandom` + +Generic body regex redactor: +- `".*([pP]assword|[Kk]ube[Cc]onfig|[Tt]oken).*"` + +#### Cluster Driver + +By default, any API request with fields tied to cluster drivers will have any non `public*` or `optional*` fields redacted by the audit log controller. + +#### Redacted URIs + +Any endpoint containing `secrets` or `configmaps` will have relevant fields stripped from both the request and response bodies. Additionally, any endpoint containing `/v3/imports/*` will have its URI redacted. + ## Viewing API Audit Logs @@ -71,492 +198,488 @@ After you enable auditing, each API request or response is logged by Rancher in ### Metadata Level -If you set your `AUDIT_LEVEL` to `1`, Rancher logs the metadata header for every API request, but not the body. The header provides basic information about the API transaction, such as the transaction's ID, who initiated the transaction, the time it occurred, etc. +If you set your `AUDIT_LEVEL` to `0`, Rancher logs the metadata header for every API request, but neither the body nor the request and response headers. The metadata provides basic information about the API transaction, such as the transaction ID, the initiator of the transaction, the time it occurred, etc. ```json { - "auditID": "30022177-9e2e-43d1-b0d0-06ef9d3db183", - "requestURI": "/v3/schemas", - "sourceIPs": ["::1"], - "user": { - "name": "user-f4tt2", - "group": ["system:authenticated"] - }, - "verb": "GET", - "stage": "RequestReceived", - "stageTimestamp": "2018-07-20 10:22:43 +0800" + "auditID": "40bd4e40-875b-4020-933e-4c4f4c4db366", + "requestURI": "/v3/schemas", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] + } + }, + "method": "GET", + "remoteAddr": "127.0.0.1:58652", + "responseCode": 200, + "requestTimestamp": "2025-06-30T11:13:25-04:00", + "responseTimestamp": "2025-06-30T11:13:25-04:00" } ``` -### Metadata and Request Body Level +### Metadata and headers level -If you set your `AUDIT_LEVEL` to `2`, Rancher logs the metadata header and body for every API request. - -The code sample below depicts an API request, with both its metadata header and body. +If you set your `AUDIT_LEVEL` to `1`, Rancher logs the metadata and the request and response headers for every API request. ```json { - "auditID": "ef1d249e-bfac-4fd0-a61f-cbdcad53b9bb", - "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "sourceIPs": ["::1"], - "user": { - "name": "user-f4tt2", - "group": ["system:authenticated"] - }, - "verb": "PUT", - "stage": "RequestReceived", - "stageTimestamp": "2018-07-20 10:28:08 +0800", - "requestBody": { - "hostIPC": false, - "hostNetwork": false, - "hostPID": false, - "paused": false, - "annotations": {}, - "baseType": "workload", - "containers": [ - { - "allowPrivilegeEscalation": false, - "image": "nginx", - "imagePullPolicy": "Always", - "initContainer": false, - "name": "nginx", - "ports": [ - { - "containerPort": 80, - "dnsName": "nginx-nodeport", - "kind": "NodePort", - "name": "80tcp01", - "protocol": "TCP", - "sourcePort": 0, - "type": "/v3/project/schemas/containerPort" - } + "auditID": "f8c83dc6-a080-4e2e-ab43-552bddf01716", + "requestURI": "/v1/apps.deployments?page=1&pagesize=100&sort=metadata.name&filter=metadata.namespace!=p-npsl5&filter=metadata.namespace!=p-nzp6c&filter=metadata.namespace!=cattle-fleet-clusters-system&filter=metadata.namespace!=cattle-fleet-system&filter=metadata.namespace!=cattle-global-data&filter=metadata.namespace!=cattle-impersonation-system&filter=metadata.namespace!=cattle-provisioning-capi-system&filter=metadata.namespace!=cattle-system&filter=metadata.namespace!=cattle-ui-plugin-system&filter=metadata.namespace!=cluster-fleet-local-local-1a3d67d0a899&filter=metadata.namespace!=fleet-default&filter=metadata.namespace!=fleet-local&filter=metadata.namespace!=kube-node-lease&filter=metadata.namespace!=kube-public&filter=metadata.namespace!=kube-system&exclude=metadata.managedFields", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" ], - "privileged": false, - "readOnly": false, - "resources": { - "type": "/v3/project/schemas/resourceRequirements", - "requests": {}, - "limits": {} - }, - "restartCount": 0, - "runAsNonRoot": false, - "stdin": true, - "stdinOnce": false, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "tty": true, - "type": "/v3/project/schemas/container", - "environmentFrom": [], - "capAdd": [], - "capDrop": [], - "livenessProbe": null, - "volumeMounts": [] - } - ], - "created": "2018-07-18T07:34:16Z", - "createdTS": 1531899256000, - "creatorId": null, - "deploymentConfig": { - "maxSurge": 1, - "maxUnavailable": 0, - "minReadySeconds": 0, - "progressDeadlineSeconds": 600, - "revisionHistoryLimit": 10, - "strategy": "RollingUpdate" - }, - "deploymentStatus": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2018-07-18T07:34:38Z", - "lastTransitionTimeTS": 1531899278000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - }, - { - "lastTransitionTime": "2018-07-18T07:34:16Z", - "lastTransitionTimeTS": 1531899256000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", - "reason": "NewReplicaSetAvailable", - "status": "True", - "type": "Progressing" + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "type": "/v3/project/schemas/deploymentStatus", - "unavailableReplicas": 0, - "updatedReplicas": 1 }, - "dnsPolicy": "ClusterFirst", - "id": "deployment:default:nginx", - "labels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "name": "nginx", - "namespaceId": "default", - "projectId": "c-bcz5t:p-fdr4s", - "publicEndpoints": [ - { - "addresses": ["10.64.3.58"], - "allNodes": true, - "ingressId": null, - "nodeId": null, - "podId": null, - "port": 30917, - "protocol": "TCP", - "serviceId": "default:nginx-nodeport", - "type": "publicEndpoint" - } - ], - "restartPolicy": "Always", - "scale": 1, - "schedulerName": "default-scheduler", - "selector": { - "matchLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "type": "/v3/project/schemas/labelSelector" - }, - "state": "active", - "terminationGracePeriodSeconds": 30, - "transitioning": "no", - "transitioningMessage": "", - "type": "deployment", - "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", - "workloadAnnotations": { - "deployment.kubernetes.io/revision": "1", - "field.cattle.io/creatorId": "user-f4tt2" - }, - "workloadLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "scheduling": { - "node": {} - }, - "description": "my description", - "volumes": [] - } -} -``` - -### Metadata, Request Body, and Response Body Level - -If you set your `AUDIT_LEVEL` to `3`, Rancher logs: - -- The metadata header and body for every API request. -- The metadata header and body for every API response. - -#### Request - -The code sample below depicts an API request, with both its metadata header and body. - -```json -{ - "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", - "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "sourceIPs": ["::1"], - "user": { - "name": "user-f4tt2", - "group": ["system:authenticated"] - }, - "verb": "PUT", - "stage": "RequestReceived", - "stageTimestamp": "2018-07-20 10:33:06 +0800", - "requestBody": { - "hostIPC": false, - "hostNetwork": false, - "hostPID": false, - "paused": false, - "annotations": {}, - "baseType": "workload", - "containers": [ - { - "allowPrivilegeEscalation": false, - "image": "nginx", - "imagePullPolicy": "Always", - "initContainer": false, - "name": "nginx", - "ports": [ - { - "containerPort": 80, - "dnsName": "nginx-nodeport", - "kind": "NodePort", - "name": "80tcp01", - "protocol": "TCP", - "sourcePort": 0, - "type": "/v3/project/schemas/containerPort" - } + "method": "GET", + "remoteAddr": "127.0.0.1:58833", + "responseCode": 200, + "requestTimestamp": "2025-06-30T11:17:04-04:00", + "responseTimestamp": "2025-06-30T11:17:04-04:00", + "requestHeader": { + "Accept": [ + "application/json" ], - "privileged": false, - "readOnly": false, - "resources": { - "type": "/v3/project/schemas/resourceRequirements", - "requests": {}, - "limits": {} - }, - "restartCount": 0, - "runAsNonRoot": false, - "stdin": true, - "stdinOnce": false, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "tty": true, - "type": "/v3/project/schemas/container", - "environmentFrom": [], - "capAdd": [], - "capDrop": [], - "livenessProbe": null, - "volumeMounts": [] - } - ], - "created": "2018-07-18T07:34:16Z", - "createdTS": 1531899256000, - "creatorId": null, - "deploymentConfig": { - "maxSurge": 1, - "maxUnavailable": 0, - "minReadySeconds": 0, - "progressDeadlineSeconds": 600, - "revisionHistoryLimit": 10, - "strategy": "RollingUpdate" - }, - "deploymentStatus": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2018-07-18T07:34:38Z", - "lastTransitionTimeTS": 1531899278000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - }, - { - "lastTransitionTime": "2018-07-18T07:34:16Z", - "lastTransitionTimeTS": 1531899256000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", - "reason": "NewReplicaSetAvailable", - "status": "True", - "type": "Progressing" - } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "type": "/v3/project/schemas/deploymentStatus", - "unavailableReplicas": 0, - "updatedReplicas": 1 - }, - "dnsPolicy": "ClusterFirst", - "id": "deployment:default:nginx", - "labels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "name": "nginx", - "namespaceId": "default", - "projectId": "c-bcz5t:p-fdr4s", - "publicEndpoints": [ - { - "addresses": ["10.64.3.58"], - "allNodes": true, - "ingressId": null, - "nodeId": null, - "podId": null, - "port": 30917, - "protocol": "TCP", - "serviceId": "default:nginx-nodeport", - "type": "publicEndpoint" - } - ], - "restartPolicy": "Always", - "scale": 1, - "schedulerName": "default-scheduler", - "selector": { - "matchLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "type": "/v3/project/schemas/labelSelector" - }, - "state": "active", - "terminationGracePeriodSeconds": 30, - "transitioning": "no", - "transitioningMessage": "", - "type": "deployment", - "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", - "workloadAnnotations": { - "deployment.kubernetes.io/revision": "1", - "field.cattle.io/creatorId": "user-f4tt2" - }, - "workloadLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "scheduling": { - "node": {} - }, - "description": "my decript", - "volumes": [] - } -} -``` - -#### Response - -The code sample below depicts an API response, with both its metadata header and body. - -```json -{ - "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", - "responseStatus": "200", - "stage": "ResponseComplete", - "stageTimestamp": "2018-07-20 10:33:06 +0800", - "responseBody": { - "actionLinks": { - "pause": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=pause", - "resume": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=resume", - "rollback": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=rollback" - }, - "annotations": {}, - "baseType": "workload", - "containers": [ - { - "allowPrivilegeEscalation": false, - "image": "nginx", - "imagePullPolicy": "Always", - "initContainer": false, - "name": "nginx", - "ports": [ - { - "containerPort": 80, - "dnsName": "nginx-nodeport", - "kind": "NodePort", - "name": "80tcp01", - "protocol": "TCP", - "sourcePort": 0, - "type": "/v3/project/schemas/containerPort" - } + "Accept-Encoding": [ + "gzip, deflate, br, zstd" ], - "privileged": false, - "readOnly": false, - "resources": { - "type": "/v3/project/schemas/resourceRequirements" - }, - "restartCount": 0, - "runAsNonRoot": false, - "stdin": true, - "stdinOnce": false, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "tty": true, - "type": "/v3/project/schemas/container" - } - ], - "created": "2018-07-18T07:34:16Z", - "createdTS": 1531899256000, - "creatorId": null, - "deploymentConfig": { - "maxSurge": 1, - "maxUnavailable": 0, - "minReadySeconds": 0, - "progressDeadlineSeconds": 600, - "revisionHistoryLimit": 10, - "strategy": "RollingUpdate" + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "Connection": [ + "keep-alive" + ], + "Cookie": [ + "[redacted]" + ], + "Referer": [ + "https://localhost:8443/dashboard/c/local/explorer/apps.deployment" + ], + "Sec-Fetch-Dest": [ + "empty" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "Sec-Fetch-Site": [ + "same-origin" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0" + ], + "X-Api-Csrf": [ + "fccc690cab7b0c169b3fc6527edadef3" + ] }, - "deploymentStatus": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2018-07-18T07:34:38Z", - "lastTransitionTimeTS": 1531899278000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - }, - { - "lastTransitionTime": "2018-07-18T07:34:16Z", - "lastTransitionTimeTS": 1531899256000, - "lastUpdateTime": "2018-07-18T07:34:38Z", - "lastUpdateTimeTS": 1531899278000, - "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", - "reason": "NewReplicaSetAvailable", - "status": "True", - "type": "Progressing" - } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "type": "/v3/project/schemas/deploymentStatus", - "unavailableReplicas": 0, - "updatedReplicas": 1 - }, - "dnsPolicy": "ClusterFirst", - "hostIPC": false, - "hostNetwork": false, - "hostPID": false, - "id": "deployment:default:nginx", - "labels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "links": { - "remove": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "revisions": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/revisions", - "self": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "update": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", - "yaml": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/yaml" - }, - "name": "nginx", - "namespaceId": "default", - "paused": false, - "projectId": "c-bcz5t:p-fdr4s", - "publicEndpoints": [ - { - "addresses": ["10.64.3.58"], - "allNodes": true, - "ingressId": null, - "nodeId": null, - "podId": null, - "port": 30917, - "protocol": "TCP", - "serviceId": "default:nginx-nodeport" - } - ], - "restartPolicy": "Always", - "scale": 1, - "schedulerName": "default-scheduler", - "selector": { - "matchLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" - }, - "type": "/v3/project/schemas/labelSelector" - }, - "state": "active", - "terminationGracePeriodSeconds": 30, - "transitioning": "no", - "transitioningMessage": "", - "type": "deployment", - "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", - "workloadAnnotations": { - "deployment.kubernetes.io/revision": "1", - "field.cattle.io/creatorId": "user-f4tt2" - }, - "workloadLabels": { - "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + "responseHeader": { + "Cache-Control": [ + "no-cache, no-store, must-revalidate" + ], + "Content-Encoding": [ + "gzip" + ], + "Content-Type": [ + "application/json" + ], + "Expires": [ + "Wed 24 Feb 1982 18:42:00 GMT" + ], + "X-Api-Cattle-Auth": [ + "true" + ], + "X-Api-Schemas": [ + "https://localhost:8443/v1/schemas" + ], + "X-Content-Type-Options": [ + "nosniff" + ] } - } } ``` + +### Metadata, headers and Request Body Level + +If you set your `AUDIT_LEVEL` to `2`, Rancher logs the metadata, the request and response headers and the request body for every API request. + +The code sample below depicts an API request, with both its metadata, headers and request body. + +```json +{ + "auditID": "d1088a09-2a13-4450-970e-0d44bd2c49ee", + "requestURI": "/v3/projects", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] + } + }, + "method": "POST", + "remoteAddr": "127.0.0.1:49966", + "responseCode": 201, + "requestTimestamp": "2025-06-30T12:32:13-04:00", + "responseTimestamp": "2025-06-30T12:32:13-04:00", + "requestHeader": { + "Accept": [ + "application/json" + ], + "Accept-Encoding": [ + "gzip, deflate, br, zstd" + ], + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "214" + ], + "Content-Type": [ + "application/json" + ], + "Cookie": [ + "[redacted]" + ], + "Impersonate-Extra-Principalid": [ + "local://user-6j5s6" + ], + "Impersonate-Extra-Requesthost": [ + "localhost:8443" + ], + "Impersonate-Extra-Requesttokenid": [ + "token-zs42h" + ], + "Impersonate-Extra-Username": [ + "admin" + ], + "Impersonate-Group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "Impersonate-User": [ + "user-6j5s6" + ], + "Origin": [ + "https://localhost:8443" + ], + "Priority": [ + "u=0" + ], + "Referer": [ + "https://localhost:8443/dashboard/c/local/explorer/management.cattle.io.project/create" + ], + "Sec-Fetch-Dest": [ + "empty" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "Sec-Fetch-Site": [ + "same-origin" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0" + ], + "X-Api-Csrf": [ + "fccc690cab7b0c169b3fc6527edadef3" + ] + }, + "responseHeader": { + "Cache-Control": [ + "no-cache, no-store, must-revalidate" + ], + "Content-Encoding": [ + "gzip" + ], + "Content-Type": [ + "application/json" + ], + "Expires": [ + "Wed 24 Feb 1982 18:42:00 GMT" + ], + "X-Api-Cattle-Auth": [ + "true" + ], + "X-Api-Schemas": [ + "https://localhost:8443/v3/project/schemas" + ], + "X-Content-Type-Options": [ + "nosniff" + ] + }, + "requestBody": { + "annotations": {}, + "clusterId": "local", + "containerDefaultResourceLimit": {}, + "creatorId": "local://user-6j5s6", + "labels": {}, + "name": "example-project", + "namespaceDefaultResourceQuota": {}, + "resourceQuota": {}, + "type": "project" + } +} +``` + +### Metadata, Headers, Request Body and Response Body Level + +If you set your `AUDIT_LEVEL` to `3`, Rancher logs the metadata, request and response headers and request body and response. + +The code sample below depicts an example of an API request with that information logged. + +```json +{ + "auditID": "a9549a5b-4351-4bd5-adcd-12f7ec667a6b", + "requestURI": "/v3/projects", + "user": { + "name": "user-6j5s6", + "group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "extra": { + "principalid": [ + "local://user-6j5s6" + ], + "requesthost": [ + "localhost:8443" + ], + "requesttokenid": [ + "token-zs42h" + ], + "username": [ + "admin" + ] + } + }, + "method": "POST", + "remoteAddr": "127.0.0.1:50454", + "responseCode": 201, + "requestTimestamp": "2025-06-30T12:42:24-04:00", + "responseTimestamp": "2025-06-30T12:42:24-04:00", + "requestHeader": { + "Accept": [ + "application/json" + ], + "Accept-Encoding": [ + "gzip, deflate, br, zstd" + ], + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "214" + ], + "Content-Type": [ + "application/json" + ], + "Cookie": [ + "[redacted]" + ], + "Impersonate-Extra-Principalid": [ + "local://user-6j5s6" + ], + "Impersonate-Extra-Requesthost": [ + "localhost:8443" + ], + "Impersonate-Extra-Requesttokenid": [ + "token-zs42h" + ], + "Impersonate-Extra-Username": [ + "admin" + ], + "Impersonate-Group": [ + "system:authenticated", + "system:cattle:authenticated" + ], + "Impersonate-User": [ + "user-6j5s6" + ], + "Origin": [ + "https://localhost:8443" + ], + "Priority": [ + "u=0" + ], + "Referer": [ + "https://localhost:8443/dashboard/c/local/explorer/management.cattle.io.project/create" + ], + "Sec-Fetch-Dest": [ + "empty" + ], + "Sec-Fetch-Mode": [ + "cors" + ], + "Sec-Fetch-Site": [ + "same-origin" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0" + ], + "X-Api-Csrf": [ + "fccc690cab7b0c169b3fc6527edadef3" + ] + }, + "responseHeader": { + "Cache-Control": [ + "no-cache, no-store, must-revalidate" + ], + "Content-Encoding": [ + "gzip" + ], + "Content-Type": [ + "application/json" + ], + "Expires": [ + "Wed 24 Feb 1982 18:42:00 GMT" + ], + "X-Api-Cattle-Auth": [ + "true" + ], + "X-Api-Schemas": [ + "https://localhost:8443/v3/project/schemas" + ], + "X-Content-Type-Options": [ + "nosniff" + ] + }, + "requestBody": { + "annotations": {}, + "clusterId": "local", + "containerDefaultResourceLimit": {}, + "creatorId": "local://user-6j5s6", + "labels": {}, + "name": "example-project", + "namespaceDefaultResourceQuota": {}, + "resourceQuota": {}, + "type": "project" + }, + "responseBody": { + "actions": { + "exportYaml": "https://localhost:8443/v3/projects/local:p-qt6tq?action=exportYaml" + }, + "annotations": { + "authz.management.cattle.io/creator-role-bindings": "{\"required\":[\"project-owner\"]}" + }, + "backingNamespace": "local-p-qt6tq", + "baseType": "project", + "clusterId": "local", + "containerDefaultResourceLimit": { + "type": "/v3/schemas/containerResourceLimit" + }, + "created": "2025-06-30T16:42:24Z", + "createdTS": 1751301744000, + "creatorId": "user-6j5s6", + "id": "local:p-qt6tq", + "labels": { + "cattle.io/creator": "norman" + }, + "links": { + "basicAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/basicauths", + "certificates": "https://localhost:8443/v3/projects/local:p-qt6tq/certificates", + "configMaps": "https://localhost:8443/v3/projects/local:p-qt6tq/configmaps", + "cronJobs": "https://localhost:8443/v3/projects/local:p-qt6tq/cronjobs", + "daemonSets": "https://localhost:8443/v3/projects/local:p-qt6tq/daemonsets", + "deployments": "https://localhost:8443/v3/projects/local:p-qt6tq/deployments", + "dnsRecords": "https://localhost:8443/v3/projects/local:p-qt6tq/dnsrecords", + "dockerCredentials": "https://localhost:8443/v3/projects/local:p-qt6tq/dockercredentials", + "horizontalPodAutoscalers": "https://localhost:8443/v3/projects/local:p-qt6tq/horizontalpodautoscalers", + "ingresses": "https://localhost:8443/v3/projects/local:p-qt6tq/ingresses", + "jobs": "https://localhost:8443/v3/projects/local:p-qt6tq/jobs", + "namespacedBasicAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedbasicauths", + "namespacedCertificates": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedcertificates", + "namespacedDockerCredentials": "https://localhost:8443/v3/projects/local:p-qt6tq/namespaceddockercredentials", + "namespacedSecrets": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedsecrets", + "namespacedServiceAccountTokens": "[redacted]", + "namespacedSshAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/namespacedsshauths", + "persistentVolumeClaims": "https://localhost:8443/v3/projects/local:p-qt6tq/persistentvolumeclaims", + "pods": "https://localhost:8443/v3/projects/local:p-qt6tq/pods", + "projectNetworkPolicies": "https://localhost:8443/v3/projects/local:p-qt6tq/projectnetworkpolicies", + "projectRoleTemplateBindings": "https://localhost:8443/v3/projects/local:p-qt6tq/projectroletemplatebindings", + "remove": "https://localhost:8443/v3/projects/local:p-qt6tq", + "replicaSets": "https://localhost:8443/v3/projects/local:p-qt6tq/replicasets", + "replicationControllers": "https://localhost:8443/v3/projects/local:p-qt6tq/replicationcontrollers", + "secrets": "https://localhost:8443/v3/projects/local:p-qt6tq/secrets", + "self": "https://localhost:8443/v3/projects/local:p-qt6tq", + "serviceAccountTokens": "[redacted]", + "services": "https://localhost:8443/v3/projects/local:p-qt6tq/services", + "sshAuths": "https://localhost:8443/v3/projects/local:p-qt6tq/sshauths", + "statefulSets": "https://localhost:8443/v3/projects/local:p-qt6tq/statefulsets", + "subscribe": "https://localhost:8443/v3/projects/local:p-qt6tq/subscribe", + "update": "https://localhost:8443/v3/projects/local:p-qt6tq", + "workloads": "https://localhost:8443/v3/projects/local:p-qt6tq/workloads" + }, + "name": "example-project", + "namespaceDefaultResourceQuota": { + "limit": { + "type": "/v3/schemas/resourceQuotaLimit" + }, + "type": "/v3/schemas/namespaceResourceQuota" + }, + "namespaceId": null, + "resourceQuota": { + "limit": { + "type": "/v3/schemas/resourceQuotaLimit" + }, + "type": "/v3/schemas/projectResourceQuota", + "usedLimit": { + "type": "/v3/schemas/resourceQuotaLimit" + } + }, + "state": "active", + "transitioning": "no", + "transitioningMessage": "", + "type": "project", + "uuid": "b582603b-7826-4302-8393-792df2611265" + } +} +``` \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md deleted file mode 100644 index f3faaf47ceb..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: UI Server-Side Pagination ---- - - - - - -:::caution -UI server-side pagination is not intended for use in production at this time. This feature is considered highly experimental. SUSE customers should consult SUSE Support before activating this feature. -::: - - -UI server-side pagination caching provides an optional SQLite-backed cache of Kubernetes objects to improve performance. This unlocks sorting, filtering and pagination features used by the UI to restrict the amount of resources it fetches and stores in browser memory. These features are primarily used to improve list performance for resources with high counts. - -This feature creates file system based caches in the `rancher` pods of the upstream cluster, and in the `cattle-cluster-agent` pods of the downstream clusters. In most environments, disk usage and I/O should not be significant. However, you should monitor activity after you enable caching. - -SQLite-backed caching persists copies of any cached Kubernetes objects to disk. See [Encrypting SQLite-backed Caching](#encrypting-sqlite-backed-caches) if this is a security concern. - -## Enabling UI Server-Side Pagination - -1. In the upper left corner, click **☰ > Global Settings > Feature Flags**. -1. Find **`ui-sql-cache`** and select **⋮ > Activate > Activate**. -1. Wait for Rancher to restart. This also restarts agents on all downstream clusters. -1. In the upper left corner, click **☰ > Global Settings > Performance**. -1. Go to **Server-side Pagination** and check the **Enable Server-side Pagination** option. -1. Click **Apply**. -1. Reload the page with the browser button (or the equivalent keyboard combination, typically `CTRL + R` on Windows and Linux, and `⌘ + R` on macOS). - - -## Encrypting SQLite-backed Caches - -UI server-side pagination persists copies of any cached Kubernetes objects to disk. If you're concerned about the safety of this data, you can encrypt all objects before they are persisted to disk, by setting the environment variable `CATTLE_ENCRYPT_CACHE_ALL` to `true` in `rancher` pods in the upstream cluster and `cattle-cluster-agent` pods in the downstream clusters. - -Secrets and security Tokens are always encrypted regardless of the above setting. - -## Known Limitations of UI Server-Side Pagination - -This initial release improves the performance of Pods, Secrets, Nodes and ConfigMaps in the Cluster Explorer pages, and most resources in the Explorer's **More Resources** section. - -Pages can't be automatically refreshed. You can manually refresh table contents by clicking the **Refresh** button. diff --git a/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/ui-server-side-pagination.md b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/ui-server-side-pagination.md new file mode 100644 index 00000000000..d9bdef91c50 --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/advanced-user-guides/ui-server-side-pagination.md @@ -0,0 +1,80 @@ +--- +title: UI Server-Side Pagination +--- + + + + + +Server-Side Pagination (SSP) is a Rancher feature to provide significant performance improvements across the UI for resources with high counts, restricting the amount of resources browser fetches and stores in memory. + +Note that SSP is optional, **enabled by default**, and it can be disabled via the feature flag `ui-sql-cache`. + +## Disk Space + +:::important +It is crucial that you review the available disk space on your nodes and plan accordingly before upgrading to Rancher v2.12.0 and later to avoid potential disk pressure and pod eviction issues. +::: + +The SSP relies on a caching mechanism that introduces a new requirement for ephemeral disk space on your cluster nodes. This cache, an internal SQLite database, is stored within the container's file system. This affects the nodes running the **Rancher server pods** (`rancher` in the `cattle-system` namespace on the local cluster) and the nodes running the **Rancher agent pods** (`cattle-cluster-agent` in the `cattle-system` namespace on all downstream clusters). + +The amount of disk space required is dynamic and depends on the quantity and size of Kubernetes resources visualized in the UI. As a guideline, the cache may consume approximately **twice the size of the raw Kubernetes objects** it stores. + +For example, internal tests showed that caching 5000 ConfigMaps, totaling 50 MB, consumed 81 MB of disk space. For a conservative, high-level estimate, you can plan for the available disk space on each relevant node to be at least **twice the size of your etcd snapshot**. For most production environments, ensuring a few extra gigabytes of storage are available on the relevant nodes is a safe starting point. + +Please note this space counts against [ephemeral storage](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#setting-requests-and-limits-for-local-ephemeral-storage) requests and limits you might have set for your Rancher container via the `resource` value in the Helm chart. Make sure those settings provide for abundant available space. + +If you see the error `database or disk is full (13)` in the pod logs, this is a symptom that more space needs to be allocated. + +SQLite-backed caching persists copies of any cached Kubernetes objects to disk. See [Encrypting SQLite-backed Caching](#encrypting-sqlite-backed-caches) if this is a security concern. + +## Enabling Server-Side Pagination + +1. In the upper left corner, click **☰ > Global Settings > Feature Flags**. +1. Find **`ui-sql-cache`** and select **⋮ > Activate > Activate**. +1. Wait for Rancher to restart. This also restarts agents on all downstream clusters. +1. Reload the page with the browser button (or the equivalent keyboard combination, typically `CTRL + R` on Windows and Linux, and `⌘ + R` on macOS). + +## Disabling Server-Side Pagination + +1. In the upper left corner, click **☰ > Global Settings > Feature Flags**. +1. Find **`ui-sql-cache`** and select **⋮ > Deactivate > Deactivate**. +1. Wait for Rancher to restart. This also restarts agents on all downstream clusters. +1. Reload the page with the browser button (or the equivalent keyboard combination, typically `CTRL + R` on Windows and Linux, and `⌘ + R` on macOS). + +## Encrypting SQLite-backed Caches + +UI server-side pagination persists copies of any cached Kubernetes objects to disk. If you're concerned about the safety of this data, you can encrypt all objects before they are persisted to disk, by setting the environment variable `CATTLE_ENCRYPT_CACHE_ALL` to `true` in `rancher` pods in the upstream cluster and `cattle-cluster-agent` pods in the downstream clusters. + +Secrets and security Tokens are always encrypted regardless of the above setting. + +## Known Limitations of UI Server-Side Pagination + +This release improves the performance of most pages used to view, create or edit resources within the `local` or downstream clusters i.e. the Cluster Explorer view. However, RBAC related resources and areas outside the Cluster Explorer are not yet covered by this feature. + +Additionally, the following limitations are present when the feature is enabled. These mainly revolve around different sort or filter behaviors in affected lists: + +- Resources in lists are automatically updated, however, not instantaneously. +- All lists that utilize Server-Side Pagination: + - `State` column sort and filter features work on the resources `metadata.state.name` field instead of one deduced locally by the UI. + - Updates are shown every 5 seconds, rather than instantly. +- Cluster Explorer: + - `Cluster` group --> `Nodes` page + - The following columns are not sortable or filterable: `Roles`, `External/Internal IP`, `CPU`, `RAM` (logic to determine their value is calculated in the browser) + - `Workloads` list: + - The `Workloads` list, which showed multiple different resource types has been removed. + - Server-Side Pagination of multiple resources is not currently possible. + - `Workloads` group --> All lists + - `Pod Restarts` and `Workload Health` columns have been removed. + - [Re-enable Pod Restart Count and Pod Health columns for Workload lists #14211](https://github.com/rancher/dashboard/issues/14211) + - `Workloads` group / `Job` List + - `Duration` is not sortable (sorting on a duration). + - [Implement more complex server-side pagination sorting #12815](https://github.com/rancher/dashboard/issues/12815) + - `Workloads` group / `Pod` List + - `Images` is not sortable (sorting on an array). + - `Service Discovery` group / `Ingresses` + - `Default` is not sortable/filterable (logic to determine their value is calculated in the browser). + - `Storage` group / `ConfigMaps` + - `Data` is not sortable/filterable (logic to determine their value is calculated in the browser). + - `Storage` group / `Secrets` + - `Data` is not sortable/filterable (logic to determine their value is calculated in the browser). diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md index 2295d1cf196..63b78e20f4c 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/about-provisioning-drivers.md @@ -49,3 +49,4 @@ Rancher supports several major cloud providers, but by default, these node drive There are several other node drivers that are disabled by default, but are packaged in Rancher: * [Harvester](../../../../integrations-in-rancher/harvester/overview.md#harvester-node-driver/), available as of Rancher v2.6.1 +* [Google GCE](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md), available as of Rancher v2.12.0 diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito.md new file mode 100644 index 00000000000..58a82dfc38d --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito.md @@ -0,0 +1,60 @@ +--- +title: Configure Amazon Cognito +description: Create an Amazon Cognito user pool and configure Rancher to work with Amazon Cognito. Your users can then sign into Rancher using their login from Amazon Cognito. +--- + + + + + +If your organization uses Amazon Cognito for user authentication, you can configure Rancher to allow login using Amazon Cognito credentials. The following instructions describe how to configure Rancher to work with Amazon Cognito: + +## Prerequisites + +- In Rancher: + - Amazon Cognito is disabled. + +:::note +Consult the Amazon Cognito [documentation](https://aws.amazon.com/cognito/getting-started/) to configure the user pool. +::: + +- In Amazon Cognito: + - Create a new user pool or use an existing one. + - In the `App client` settings, set the redirect URL to `https://yourRancherHostURL/verify-auth`. Replace `yourRancherHostURL` with the actual hostname of your Rancher instance (e.g., https://rancher.example.com/verify-auth). + +## Configuring Amazon Cognito in Rancher + +1. In the upper left corner of the Rancher UI, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. Select **Amazon Cognito**. +1. Complete the **Configure an Amazon Cognito account** form. For help with filling the form, see the [configuration reference](#configuration-reference). +1. Click **Enable**. + + Rancher will redirect you to the Amazon Cognito login page. Enter your Amazon Cognito credentials to validate your Rancher configuration. + + :::note + + You may need to disable your popup blocker to see the Amazon Cognito login page. + + ::: + +**Result:** Rancher is configured to work with your Amazon Cognito using the OIDC protocol. Your users can now sign into Rancher using their Amazon Cognito logins. + +:::note +User and group search is not supported for Amazon Cognito. When assigning permissions to a Project or Cluster, you must manually enter the UserID generated by Cognito +if the user has not yet logged in to Rancher. However, if the user has previously logged in, you can assign permissions using their username or email address. +::: + +## Configuration Reference + +| Field | Description | +| ------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Client ID | The Client ID of your Amazon Cognito App Client. | +| Client Secret | The generated Secret of your Amazon Cognito App Client. | +| Issuer | The Issuer URL of your Amazon Cognito App Client. It follows the format `https://cognito-idp.{region}.amazonaws.com/{userPoolId}`, and can be found in the App Client settings page. Rancher uses the Issuer URL to fetch all of the required URLs. | + +## Troubleshooting + +### You are not redirected to your authentication provider + +If you fill out the **Configure an Amazon Cognito account** form and click on **Enable**, and you are not redirected to Amazon Cognito, verify your Amazon Cognito configuration. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md index 22aab5d5cac..c896b2ebf16 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md @@ -196,6 +196,34 @@ As previously mentioned, custom roles can be defined for use at the cluster or p When defining a custom role, you can grant access to specific resources or specify roles from which the custom role should inherit. A custom role can be made up of a combination of specific grants and inherited roles. All grants are additive. This means that defining a narrower grant for a specific resource **will not** override a broader grant defined in a role that the custom role is inheriting from. +#### UpdatePSA For Project Level + +About defining custom roles, you can grant permission to a user to create or update *PSA* policies when defining namespaces within projects. + +To do so, you can use the following `RoleTemplate` to be applied on the cluster: + +```yaml +apiVersion: management.cattle.io/v3 +builtin: false +context: project +description: '' +displayName: Manage PSA Labels +external: false +hidden: false +kind: RoleTemplate +metadata: + name: namespaces-psa +rules: + - apiGroups: + - management.cattle.io + resources: + - projects + verbs: + - updatepsa +``` + +When creating a new project (from the **Members** tab), click **Add** to add the user and select **Custom** > **Create Namespaces** (to allow the user to create namespaces). Then click **Add** again and select `UpdatePSA` project role template from the list of **Project Permissions**.gt + ### Default Cluster and Project Roles By default, when a standard user creates a new cluster or project, they are automatically assigned an ownership role: either [cluster owner](#cluster-roles) or [project owner](#project-roles). However, in some organizations, these roles may overextend administrative access. In this use case, you can change the default role to something more restrictive, such as a set of individual roles or a custom role. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center.md new file mode 100644 index 00000000000..0c049f2076b --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center.md @@ -0,0 +1,23 @@ +--- +title: Notification Center +--- + + + + + +## What is the Notification Center? + +The Notification Center, located in the upper-right corner of your Rancher dashboard and marked by a bell icon, is your central hub for staying informed about various events within Rancher. + +Notifications are categorized by severity and type: + +* **Error** indicates a high-severity issue. +* **Warning** indicates a medium-severity concern. +* **Information** provides general updates of lower severity. +* **Task** shows that a process or action is currently in progress. +* **Success** confirms that a process or action has completed successfully. + +For example, you might receive a notification when a new version of Rancher is available, or a direct link to the Rancher Release Notes. + +You can easily browse through your notifications using the up and down arrow keys. Certain notifications may also include an associated action, allowing you to respond immediately. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md index 0ea24911d5a..d6d2942afd2 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md @@ -38,9 +38,6 @@ You can assign a PSA template at the same time that you create a downstream clus 1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. 1. Click **Save**. -### Hardening the Cluster - -If you select the **rancher-restricted** template but don't select a **CIS Profile**, you won't meet required CIS benchmarks. See the [RKE2 hardening guide](../../../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md) for more details. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md index 9d3a6307a5f..2edb0398c8c 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md @@ -70,7 +70,7 @@ To perform a backup, a custom resource of type Backup must be created. folder: rancher region: us-west-2 endpoint: s3.us-west-2.amazonaws.com - resourceSetName: rancher-resource-set + resourceSetName: rancher-resource-set-full encryptionConfigSecretName: encryptionconfig schedule: "@every 1h" retentionCount: 10 @@ -78,7 +78,7 @@ To perform a backup, a custom resource of type Backup must be created. :::note - When creating the Backup resource using YAML editor, the `resourceSetName` must be set to `rancher-resource-set` + When creating the Backup resource using YAML editor, the `resourceSetName` must be set to `rancher-resource-set-full` or `rancher-resource-set-basic`. ::: diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md index 71467ac7403..c3b26afd434 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/backup-restore-and-disaster-recovery.md @@ -28,7 +28,7 @@ The `rancher-backup` operator introduces three custom resources: Backups, Restor The ResourceSet defines which Kubernetes resources need to be backed up. The ResourceSet is not available to be configured in the Rancher UI because the values required to back up Rancher are predefined. This ResourceSet should not be modified. -When a Backup custom resource is created, the `rancher-backup` operator calls the `kube-apiserver` to get the resources in the ResourceSet (specifically, the predefined `rancher-resource-set`) that the Backup custom resource refers to. +When a Backup custom resource is created, the `rancher-backup` operator calls the `kube-apiserver` to get the resources in the ResourceSet that the Backup custom resource refers to. The operator then creates the backup file in the .tar.gz format and stores it in the location configured in the Backup resource. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md index 844fbaad30f..20d09a90ca0 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -170,6 +170,13 @@ You can define the default behavior for newly created clusters or existing ones Changes to the global **imported-cluster-version-management** setting take effect during the cluster’s next reconciliation cycle. +:::note + +If version management is enabled for a cluster, Rancher will deploy the system-upgrade-controller app, along with the associated Plans and other required Kubernetes resources, to the cluster. +If version management is disabled, Rancher will remove these components from the cluster. + +::: + ## Configuring RKE2 and K3s Cluster Upgrades :::tip diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md index 89e4a40ba5d..362ffac61e5 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md @@ -14,15 +14,16 @@ This page is about secrets in general. For details on setting up a private regis ::: -When configuring a workload, you'll be able to choose which secrets to include. Like config maps, secrets can be referenced by workloads as either an environment variable or a volume mount. +When configuring a workload, you are able to choose which secrets to include. Like config maps, secrets can be referenced by workloads as either an environment variable or a volume mount. -Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) +Mounted secrets are updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) ## Creating Secrets in Namespaces 1. In the upper left corner, click **☰ > Cluster Management**. 1. Go to the cluster where you want to add a secret and click **Explore**. 1. To navigate to secrets, you may click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Select the **Namespaced** tab. 1. Click **Create**. 1. Select the type of secret you want to create. 1. Select a **Namespace** for the secret. @@ -48,32 +49,66 @@ Mounted secrets will be updated automatically unless they are mounted as subpath **Result:** Your secret is added to the namespace you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. -Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) +Mounted secrets are updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) ## Creating Secrets in Projects -Before v2.6, secrets were required to be in a project scope. Projects are no longer required, and you may use the namespace scope instead. As a result, the Rancher UI was updated to reflect this new functionality. However, you may still create project-scoped secrets if desired. Note that you have to first enable the `legacy` feature flag and look at a single project to do so. Use the following steps to set up your project-level secret: +When creating a secret in a project scope, the secret is copied into all namespaces within the project. -1. In the upper left corner, click **☰ > Global Settings** in the dropdown. -1. Click **Feature Flags**. -1. Go to the `legacy` feature flag and click **Activate**. -1. In the upper left corner, click **☰ > Cluster Management** in the dropdown. -1. Go to the cluster that you created and click **Explore.** -1. Click **Legacy > Projects**. -1. In the top navigation bar, filter to see only one project. -1. In the left navigation bar, click **Secrets**. -1. Click **Add Secret**. +### Creating a Project Scoped Secret in the UI -**Result:** Your secret is added to the individual project you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a secret and click **Explore**. +1. To navigate to secrets, you may click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Select the **Project Scoped** tab. +1. Click **Create Project Scoped Secret**. +1. Select the type of secret you want to create. +1. Select a **Project** for the secret. +1. Enter a **Name** for the secret. -Since project-scoped secrets are set at the project level, any changes made at the namespace level will be overwritten. + :::note -:::note + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a namespace can have duplicate names. If you create a project scoped secret that has the same name as an existing secret in one of the project namespaces, the existing secret is overwritten. -Project-scoped secrets on the local cluster are only visible when a single project is selected. + ::: -::: +1. From **Data**, click **Add** to add a key-value pair. Add as many values as you need. + + :::tip + + You can add multiple key-value pairs to the secret by copying and pasting. + + ::: + + ![](/img/bulk-key-values.gif) + +1. Click **Save**. + +**Result:** Your secret is added to each namespace within the project. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. + +### Creating a Project Scoped Secret with kubectl + +Project scoped secrets work by creating the original secret on the management cluster in what's known as the "Project Backing Namespace". Rancher stores important project related information in this namespace. You can find it in the `status.backingNamespace` field in the project CRD, or by doing `kubectl get projects -A` in the management cluster. + +In order for the secret to be acknowledged by Rancher as a project scoped secret, it also needs the label `management.cattle.io/project-scoped-secret: `. + +Example yaml: + +``` +apiVersion: v1 +data: + key: ZG9n +kind: Secret +metadata: + labels: + management.cattle.io/project-scoped-secret: p-vwxyz + name: test-secret + namespace: c-abc123-p-vwxyz +type: Opaque +``` + +In the above YAML, the namespace is the backing namespace of project `p-vwxyz` and the project scoped secret label references the projectID. When applied to the management cluster, all namespaces within the project `p-vwxyz` contain a copy of `test-secret`. ## What's Next? diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md index 66010c77149..c8c2c442025 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md @@ -54,12 +54,18 @@ If you use RKE2 to set up a cluster, your nodes must meet the [requirements](htt ### Launching Kubernetes on New Nodes in an Infrastructure Provider -RKE2 provisioning is built on top of a new provisioning framework that leverages the upstream [Cluster API](https://github.com/kubernetes-sigs/cluster-api) project. With this new provisioning framework, you can: +RKE2 provisioning is built on top of a new provisioning framework that leverages the upstream [Cluster API (CAPI)](https://github.com/kubernetes-sigs/cluster-api) project. With this new provisioning framework, you can: - Provision RKE2 clusters onto any provider for which Rancher has a node driver - Fully configure RKE2 clusters within Rancher - Choose CNI options Calico, Cilium, and Multus in addition to Canal +When you make changes to your cluster configuration in RKE2, this may result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. + +The following are some specific example configuration changes that may cause the described behavior: + +- When editing the cluster and enabling drain before delete, the existing control plane nodes and worker are deleted and new nodes are created. + RKE2 provisioning also includes installing RKE2 on clusters with Windows nodes. Windows features for RKE2 include: diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md deleted file mode 100644 index ed8db366fbc..00000000000 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Behavior Differences Between RKE1 and RKE2 ---- - - - - - - - -RKE2, also known as RKE Government, is a Kubernetes distribution that focuses on security and compliance for U.S. Federal Government entities. It is considered the next iteration of the Rancher Kubernetes Engine, now known as RKE1. - -RKE1 and RKE2 have several slight behavioral differences to note, and this page will highlight some of these at a high level. - -### Control Plane Components - -RKE1 uses Docker for deploying and managing control plane components, and it also uses Docker as the container runtime for Kubernetes. By contrast, RKE2 launches control plane components as static pods that are managed by the kubelet. RKE2's container runtime is containerd, which allows things such as mirroring a container image registry. RKE1 with Docker does not allow mirroring. - -### Cluster API - -RKE2/K3s provisioning is built on top of the Cluster API (CAPI) upstream framework which often makes RKE2-provisioned clusters behave differently than RKE1-provisioned clusters. - -When you make changes to your cluster configuration in RKE2, this **may** result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. - -The following are some specific example configuration changes that may cause the described behavior: - -- When editing the cluster and enabling `drain before delete`, the existing control plane nodes and worker are deleted and new nodes are created. - -Users who are used to RKE1 provisioning should take note of this new RKE2 behavior which may be unexpected. - -### Terminology - -You will notice that some terms have changed or gone away going from RKE1 to RKE2. For example, in RKE1 provisioning, you use **node templates**; in RKE2 provisioning, you can configure your cluster node pools when creating or editing the cluster. Another example is that the term **node pool** in RKE1 is now known as **machine pool** in RKE2. - - - - diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md index 6f64d0abb70..7e6b5b57d11 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md @@ -6,53 +6,11 @@ title: Creating a DigitalOcean Cluster -In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in DigitalOcean. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in DigitalOcean. -First, you will set up your DigitalOcean cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in DigitalOcean. +First, you will set up your DigitalOcean cloud credentials in Rancher. -Then you will create a DigitalOcean cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. - - - - -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - -### 1. Create your cloud credentials - -1. Click **☰ > Cluster Management**. -1. Click **Cloud Credentials**. -1. Click **Create**. -1. Click **DigitalOcean**. -1. Enter your Digital Ocean credentials. -1. Click **Create**. - -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. - -### 2. Create a node template with your cloud credentials - -Creating a [node template](use-new-nodes-in-an-infra-provider.md#node-templates) for DigitalOcean will allow Rancher to provision new nodes in DigitalOcean. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates**. -1. Click **Add Template**. -1. Click **DigitalOcean**. -1. Fill out a node template for DigitalOcean. For help filling out the form, refer to [DigitalOcean Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md) - -### 3. Create a cluster with node pools using the node template - -1. Click **☰ > Cluster Management**. -1. On the **Clusters** page, click **Create**. -1. Click **DigitalOcean**. -1. Enter a **Cluster Name**. -1. Add one or more node pools to your cluster. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](use-new-nodes-in-an-infra-provider.md) -1. **In the Cluster Configuration** section, choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) -1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Click **Create**. - - - +Then you will create a DigitalOcean cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. ### 1. Create your cloud credentials @@ -77,13 +35,10 @@ Use Rancher to create a Kubernetes cluster in DigitalOcean. 1. Enter a **Cluster Name**. 1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. 1. For each machine pool, define the machine configuration. Refer to the [DigitalOcean machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md) for information on configuration options. -1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. 1. Click **Create**. - - - **Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md new file mode 100644 index 00000000000..d7a09f4089c --- /dev/null +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster.md @@ -0,0 +1,107 @@ +--- +title: Creating a Google Compute Engine cluster +--- + + + + + + +In this section, you'll learn how to use Rancher to provision an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster on the Google Cloud Platform (GCP) using Google Compute Engine (GCE) through Rancher. + + +First, you will enable the GCE node driver in the Rancher UI. Then, you follow the steps to create a GCP service account with the necessary permissions, and generate a JSON key file. This key file will be used to create a cloud credential in Rancher. + + +Then, you will create a GCE cluster in Rancher, and when configuring the cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE2 onto the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. + + +1. [Enable the GCE node driver](#1-enable-the-gce-node-driver) +1. [Create your cloud credential](#2-create-a-cloud-credential) +1. [Create a GCE cluster with your cloud credential](#3-create-a-cluster-using-the-cloud-credential) +1. [GCE Best Practices](#gce-best-practices) + +### Prerequisites + +1. A valid Google Cloud Platform account and project. +1. A GCP Service Account JSON key file. The service account associated with this key must have the following IAM roles: + 1. **Compute Admin** + 1. **Service Account User** + 1. **Viewer** +1. A VPC Network to provision VMs within. + +Refer to the [GCP documentation](https://cloud.google.com/iam/docs/service-account-overview) on creating and managing service account keys for more details. + + +### 1. Enable the GCE node driver + +The GCE node driver is not enabled by default in Rancher. You must enable it before you can provision GCE clusters, or work with GCE specific CRDs. + +1. Click **☰ > Cluster Management**. +1. On the left hand side, click **Drivers**. +1. Open the **Node Drivers** tab. +1. Find the **Google GCE** driver and select **⋮ > Activate**. + + +### 2. Create a cloud credential + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Google**. +1. Enter your GCP Service Account JSON key file. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials in other clusters. Depending on the permissions granted to the service account, this credential may also be used for GKE clusters. + + +### 3. Create a cluster using the cloud credential + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Google GCE**. +1. Select a **Cloud Credential** and provide the GCP project to create the VM in. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to the [Google GCE machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +### GCE Best Practices + +#### External Firewall Rules, Open Ports, and ACE + +If the cluster being provisioned will utilize the [Authorized Cluster Endpoint (ACE) feature](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster), controlplane nodes must expose port `6443`. This port is not exposed in the default machine pool configuration to prevent it from being exposed across all cluster nodes, and to reduce the number of firewall rules created by Rancher. + +In order for ACE to work as expected, you must specify this port in the Rancher UI when configuring the controlplane machine pool by enabling the `Expose external ports` checkbox, under the `Show Advanced` section of the machine pool configuration UI. Alternatively, you may manually create a custom firewall rule in GCP and provide the related network tag in the controlplane machine-pool configuration. + +#### Internal Firewall Rules + +Rancher will automatically create a firewall rule and network tag to facilitate communication between cluster nodes internally within the specified VPC network. This rule will contain the minimum number of ports required to create an RKE2/K3s cluster. + +If you need to extend the number of ports exposed internally between cluster nodes, a new firewall rule should be manually created, and the associated network tag assigned to the relevant machine pools. If desired, the automatic creation of the internal firewall rule can be disabled for each given machine pool when creating or updating the cluster. + +#### Cross Network Deployments + +While it is possible to deploy different machine pools into different VPC networks, the internal firewall rule created by Rancher does not support this configuration by default. To create machine pools in different networks, additional firewall rules to facilitate communication between nodes in different networks must be manually created. + + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md index 3a4db1d0a73..11231c88c5a 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md @@ -7,11 +7,11 @@ description: Learn the prerequisites and steps required in order for you to crea -In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Amazon EC2. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in Amazon EC2. -First, you will set up your EC2 cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in EC2. +First, you will set up your EC2 cloud credentials in Rancher. -Then you will create an EC2 cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. +Then you will create an EC2 cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. ### Prerequisites @@ -26,64 +26,6 @@ Then you will create an EC2 cluster in Rancher, and when configuring the new clu The steps to create a cluster differ based on your Rancher version. - - - -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials and information from EC2](#2-create-a-node-template-with-your-cloud-credentials-and-information-from-ec2) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - -### 1. Create your cloud credentials - -1. Click **☰ > Cluster Management**. -1. Click **Cloud Credentials**. -1. Click **Create**. -1. Click **Amazon**. -1. Enter a name for the cloud credential. -1. In the **Default Region** field, select the AWS region where your cluster nodes will be located. -1. Enter your AWS EC2 **Access Key** and **Secret Key**. -1. Click **Create**. - -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. - -### 2. Create a node template with your cloud credentials and information from EC2 - -Creating a [node template](use-new-nodes-in-an-infra-provider.md#node-templates) for EC2 will allow Rancher to provision new nodes in EC2. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates** -1. Click **Add Template**. -1. Fill out a node template for EC2. For help filling out the form, refer to [EC2 Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md) -1. Click **Create**. - - :::note - - If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. - - ::: - -### 3. Create a cluster with node pools using the node template - -Add one or more node pools to your cluster. For more information about node pools, see [this section.](use-new-nodes-in-an-infra-provider.md) - -1. Click **☰ > Cluster Management**. -1. On the **Clusters** page, click **Create**. -1. Click **Amazon EC2**. -1. Create a node pool for each Kubernetes role. For each node pool, choose a node template that you created. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](use-new-nodes-in-an-infra-provider.md) -1. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. Refer to [Selecting Cloud Providers](../../kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/set-up-cloud-providers.md) to configure the Kubernetes Cloud Provider. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) - - :::note - - If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. - - ::: - -1. Click **Create**. - - - - ### 1. Create your cloud credentials If you already have a set of cloud credentials to use, skip this section. @@ -97,7 +39,7 @@ If you already have a set of cloud credentials to use, skip this section. 1. Enter your AWS EC2 **Access Key** and **Secret Key**. 1. Click **Create**. -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. ### 2. Create your cluster @@ -109,13 +51,10 @@ If you already have a set of cloud credentials to use, skip this section. 1. Enter a **Cluster Name**. 1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. 1. For each machine pool, define the machine configuration. Refer to [the EC2 machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md) for information on configuration options. -1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. 1. Click **Create**. - - - **Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md index 2b9e2e10cdd..186a95f366c 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md @@ -6,15 +6,15 @@ title: Creating an Azure Cluster -In this section, you'll learn how to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Azure through Rancher. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in Azure through Rancher. -First, you will set up your Azure cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in Azure. +First, you will set up your Azure cloud credentials in Rancher. -Then you will create an Azure cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. +Then you will create an Azure cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. :::caution -When the Rancher RKE cluster is running in Azure and has an Azure load balancer in front, the outbound flow will fail. The workaround for this problem is as follows: +When the Rancher RKE2/K3s cluster is running in Azure and has an Azure load balancer in front, the outbound flow will fail. The workaround for this problem is as follows: - Terminate the SSL/TLS on the internal load balancer - Use the L7 load balancer @@ -23,16 +23,16 @@ For more information, refer to the documentation on [Azure load balancer limitat ::: -For more information on configuring the Kubernetes cluster that Rancher will install on the Azure nodes, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +For more information on configuring the Kubernetes cluster that Rancher will install on the Azure nodes, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration references. -For more information on configuring Azure node templates, refer to the [Azure node template configuration reference.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) +For more information on configuring Azure machines, refer to the [Azure machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md). - [Preparation in Azure](#preparation-in-azure) - [Creating an Azure Cluster](#creating-an-azure-cluster) ## Preparation in Azure -Before creating a node template in Rancher using a cloud infrastructure such as Azure, we must configure Rancher to allow the manipulation of resources in an Azure subscription. +Before a cluster can be deployed, we must configure our Azure subscription to allow the manipulation of its resources by a third party, such as Rancher. To do this, we will first create a new Azure **service principal (SP)** in Azure **Active Directory (AD)**, which, in Azure, is an application user who has permission to manage Azure resources. @@ -45,54 +45,10 @@ az ad sp create-for-rbac \ --scopes="/subscriptions/" ``` -The creation of this service principal returns three pieces of identification information, *The application ID, also called the client ID*, and *The client secret*. This information will be used when you create a node template for Azure. +The creation of this service principal returns the **application ID**, also called the **client ID**, and the **client secret**. This information is used when you create your cloud credentials. ## Creating an Azure Cluster - - - -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - -### 1. Create your cloud credentials - -1. Click **☰ > Cluster Management**. -1. Click **Cloud Credentials**. -1. Click **Create**. -1. Click **Azure**. -1. Enter your Azure credentials. -1. Click **Create**. - -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. - -### 2. Create a node template with your cloud credentials - -Creating a [node template](use-new-nodes-in-an-infra-provider.md#node-templates) for Azure will allow Rancher to provision new nodes in Azure. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates**. -1. Click **Add Template**. -1. Click **Azure**. -1. Fill out a node template for Azure. For help filling out the form, refer to [Azure Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) - -### 3. Create a cluster with node pools using the node template - -Use Rancher to create a Kubernetes cluster in Azure. - -1. Click **☰ > Cluster Management**. -1. On the **Clusters** page, click **Create**. -1. Click **Azure**. -1. Enter a **Cluster Name**. -1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices, see [this section.](use-new-nodes-in-an-infra-provider.md) -1. In the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) -1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Click **Create**. - - - - ### 1. Create your cloud credentials If you already have a set of cloud credentials to use, skip this section. @@ -104,7 +60,7 @@ If you already have a set of cloud credentials to use, skip this section. 1. Enter your Azure credentials. 1. Click **Create**. -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. ### 2. Create your cluster @@ -118,13 +74,10 @@ Use Rancher to create a Kubernetes cluster in Azure. 1. Enter a **Cluster Name**. 1. Create a machine pool for each Kubernetes role. Refer to the [best practices](use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. 1. For each machine pool, define the machine configuration. Refer to the [Azure machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) for information on configuration options. -1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. 1. Click **Create**. - - - **Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md index 8d498ac35e6..0ecdfd2e185 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/nutanix.md @@ -18,4 +18,4 @@ A Nutanix cluster may consist of multiple groups of VMs with distinct properties ## Creating a Nutanix Cluster -In [this section,](provision-kubernetes-clusters-in-aos.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Nutanix AOS. \ No newline at end of file +In [this section,](provision-kubernetes-clusters-in-aos.md) you'll learn how to use Rancher to install an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in Nutanix AOS. \ No newline at end of file diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md index f5e74b760e5..b33c13a7170 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md @@ -6,9 +6,9 @@ title: Creating a VMware vSphere Virtual Machine Template -Creating virtual machines in a repeatable and reliable fashion can often be difficult. VMware vSphere offers the ability to build one VM that can then be converted to a template. The template can then be used to create identically configured VMs. Rancher leverages this capability within node pools to create identical RKE1 and RKE2 nodes. +Creating virtual machines in a repeatable and reliable fashion can often be difficult. VMware vSphere offers the ability to build one VM that can then be converted to a template. The template can then be used to create identically configured VMs. Rancher leverages this capability to create identical RKE/K3s nodes. -In order to leverage the template to create new VMs, Rancher has some [specific requirements](#requirements) that the VM must have pre-installed. After you configure the VM with these requirements, you will next need to [prepare the VM](#preparing-your-vm) before [creating the template](#creating-a-template). Finally, once preparation is complete, the VM can be [converted to a template](#converting-to-a-template) and [moved into a content library](#moving-to-a-content-library), ready for Rancher node pool usage. +In order to leverage the template to create new VMs, Rancher has some [specific requirements](#requirements) that the VM must have pre-installed. After you configure the VM with these requirements, you will next need to [prepare the VM](#preparing-your-vm) before [creating the template](#creating-a-template). Finally, once preparation is complete, the VM can be [converted to a template](#converting-to-a-template) and [moved into a content library](#moving-to-a-content-library). ## Requirements @@ -47,14 +47,6 @@ The list of packages that need to be installed on the template is as follows: * Windows Container Feature * [cloudbase-init](https://cloudbase.it/cloudbase-init/#download) -* [Docker EE](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server#install-docker) - RKE1 Only - -:::note About the configuration for Windows templates varies between RKE1 and RKE2: - -- RKE1 leverages Docker, so any RKE1 templates need to have Docker EE pre-installed as well -- RKE2 does not require Docker EE, and thus it does not need to be installed - -::: ## Creating a Template diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md index 20eb639ad31..143bea5a219 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md @@ -34,7 +34,7 @@ The following steps create a role with the required privileges and then assign i 4. Go to the **Users and Groups** tab. -5. Create a new user. Fill out the form and then click **OK**. Make sure to note the username and password, because you will need it when configuring node templates in Rancher. +5. Create a new user. Fill out the form and then click **OK**. Make sure to note the username and password, because you will need it when creating cloud credentials in Rancher. ![](/img/rancheruser.png) diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md index 0694c461321..2373f134ed4 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md @@ -6,16 +6,11 @@ title: Provisioning Kubernetes Clusters in VMware vSphere -In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in VMware vSphere. +In this section, you'll learn how to deploy an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in VMware vSphere. -First, you will set up your vSphere cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision nodes in vSphere. - -Then you will create a vSphere cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. - -For details on configuring the vSphere node template, refer to the [vSphere node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) - -For details on configuring RKE Kubernetes clusters in Rancher, refer to the [cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) +First, you will set up your vSphere cloud credentials in Rancher. +Then you will create a vSphere cluster in Rancher, and when configuring the new cluster, you will define machine pools for it. Each machine pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the machine pool. - [Preparation in vSphere](#preparation-in-vmware-vsphere) - [Creating a vSphere Cluster](#creating-a-vmware-vsphere-cluster) @@ -24,11 +19,9 @@ For details on configuring RKE Kubernetes clusters in Rancher, refer to the [clu This section describes the requirements for setting up vSphere so that Rancher can provision VMs and clusters. -The node templates are documented and tested with the vSphere Web Services API version 6.5. - ### Create Credentials in VMware vSphere -Before proceeding to create a cluster, you must ensure that you have a vSphere user with sufficient permissions. When you set up a node template, the template will need to use these vSphere credentials. +Before proceeding to create a cluster, you must ensure that you have a vSphere user with sufficient permissions. Refer to this [how-to guide](create-credentials.md) for instructions on how to create a user in vSphere with the required permissions. These steps result in a username and password that you will need to provide to Rancher, which allows Rancher to provision resources in vSphere. @@ -58,45 +51,30 @@ User-data.iso files may have become orphaned upon node deletion due to a vSphere ::: -1. [Create your cloud credentials](#1-create-your-cloud-credentials) -2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) -3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) - ### 1. Create your cloud credentials 1. Click **☰ > Cluster Management**. 1. Click **Cloud Credentials**. 1. Click **Create**. 1. Click **VMware vSphere**. -1. Enter your vSphere credentials. For help, refer to **Account Access** in the [node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) +1. Enter your vSphere credentials. 1. Click **Create**. -**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. -### 2. Create a node template with your cloud credentials - -Creating a [node template](../use-new-nodes-in-an-infra-provider.md#node-templates) for vSphere will allow Rancher to provision new nodes in vSphere. Node templates can be reused for other clusters. - -1. Click **☰ > Cluster Management**. -1. Click **RKE1 Configuration > Node Templates**. -1. Click **Create**. -1. Click **Add Template**. -1. Click **vSphere**. -1. Fill out a node template for vSphere. For help filling out the form, refer to the vSphere node template [configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md). -1. Click **Create**. - -### 3. Create a cluster with node pools using the node template +### 2. Create your cluster Use Rancher to create a Kubernetes cluster in vSphere. 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, click **Create**. 1. Click **VMware vSphere**. -1. Enter a **Cluster Name** and use your vSphere cloud credentials. Click **Continue**. +1. Enter a **Cluster Name** and use your vSphere cloud credentials. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. +1. Use **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) and [K3s](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) cluster configuration reference. 1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. -1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) 1. If you want to dynamically provision persistent storage or other infrastructure later, you will need to enable the vSphere cloud provider by modifying the cluster YAML file. For details, refer to [in-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). -1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to the nodes, see [this section.](../use-new-nodes-in-an-infra-provider.md#node-pools) 1. Review your options to confirm they're correct. Then click **Create**. **Result:** diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md index 78c437eed81..0234eadecf4 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/shutdown-vm.md @@ -10,8 +10,6 @@ In Rancher v2.8.3 and later, you can configure the graceful shutdown of virtual In RKE2/K3s, you can set up graceful shutdown when you create the cluster, or edit the cluster configuration to add it afterward. -In RKE, you can edit node templates to similar results. - :::note Since Rancher can't detect the platform of an imported cluster, you cannot enable graceful shutdown on VMware vSphere clusters you have imported. @@ -20,30 +18,12 @@ Since Rancher can't detect the platform of an imported cluster, you cannot enabl ## Enable Graceful Shutdown During VMware vSphere Cluster Creation - - - In RKE2/K3s, you can configure new VMware vSphere clusters with graceful shutdown for VMs: 1. Click **☰ > Cluster Management**. 1. Click **Create** and select **VMware vSphere** to provision a new cluster. 1. Under **Machine Pools > Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. - - - -In RKE, you can't directly configure a new cluster with graceful shutdown. However, you can configure node templates which automatically create node pools with graceful shutdown enabled. The node template can then be used to provision new VMware vSphere clusters that have a graceful shutdown delay. - -1. Click **☰ > Cluster Management**. -1. From the left navigation, select **RKE1 Configuration > Node Templates**. -1. Click **Add Template** and select **vSphere** to create a node template. -1. Under **2. Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. - -When you [use the newly-created node template to create node pools](../use-new-nodes-in-an-infra-provider.md), the nodes will gracefully shutdown of VMs according to the **Graceful Shutdown Timeout** value you have set. - - - - ## Enable Graceful Shutdown in Existing RKE2/K3s Clusters In RKE2/K3s, you can edit the configuration of an existing VMware vSphere cluster to enable graceful shutdown, which adds a delay before deleting VMs. @@ -52,14 +32,3 @@ In RKE2/K3s, you can edit the configuration of an existing VMware vSphere cluste 1. On the **Clusters** page, find the VMware vSphere hosted cluster you want to edit. Click **⋮** at the end of the row associated with the cluster. Select **Edit Config**. 1. Under **Machine Pools > Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. -## Enable Graceful Shutdown in Existing RKE Clusters - -In RKE, you can't directly edit an existing cluster's configuration to add graceful shutdown to existing VMware vSphere clusters. However, you can edit the configuration of existing node templates. As noted in [Updating a Node Template](../../../../../reference-guides/user-settings/manage-node-templates.md#updating-a-node-template), all node pools using the node template automatically use the updated information when new nodes are added to the cluster. - -To edit an existing node template to enable graceful shutdown: - -1. Click **☰ > Cluster Management**. -1. From the left navigation, select **RKE1 Configuration > Node Templates**. -1. Find the VMware vSphere node template you want to edit. Click **⋮** at the end of the row associated with the template. Select **Edit**. -1. Under **2. Scheduling**, in the **Graceful Shutdown Timeout** field, enter an integer value greater than 0. The value you enter is the amount of time in seconds Rancher waits before deleting VMs on the cluster. If the value is set to `0`, graceful shutdown is disabled. -1. Click **Save**. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md index 553905b2a0b..f155dd8263d 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/vsphere.md @@ -15,33 +15,9 @@ Rancher can provision nodes in vSphere and install Kubernetes on them. When crea A vSphere cluster may consist of multiple groups of VMs with distinct properties, such as the amount of memory or the number of vCPUs. This grouping allows for fine-grained control over the sizing of nodes for each Kubernetes role. -## VMware vSphere Enhancements - -The vSphere node templates allow you to bring cloud operations on-premises with the following enhancements: - -### Self-healing Node Pools - -One of the biggest advantages of provisioning vSphere nodes with Rancher is that it allows you to take advantage of Rancher's self-healing node pools, also called the [node auto-replace feature,](../use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) in your on-premises clusters. Self-healing node pools are designed to help you replace worker nodes for stateless applications. When Rancher provisions nodes from a node template, Rancher can automatically replace unreachable nodes. - -:::caution - -It is not recommended to enable node auto-replace on a node pool of master nodes or nodes with persistent volumes attached, because VMs are treated ephemerally. When a node in a node pool loses connectivity with the cluster, its persistent volumes are destroyed, resulting in data loss for stateful applications. - -::: - -### Dynamically Populated Options for Instances and Scheduling - -Node templates for vSphere have been updated so that when you create a node template with your vSphere credentials, the template is automatically populated with the same options for provisioning VMs that you have access to in the vSphere console. - -For the fields to be populated, your setup needs to fulfill the [prerequisites.](provision-kubernetes-clusters-in-vsphere.md#preparation-in-vmware-vsphere) - -### More Supported Operating Systems - -You can provision VMs with any operating system that supports `cloud-init`. Only YAML format is supported for the [cloud config.](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) - ## Creating a VMware vSphere Cluster -In [this section,](provision-kubernetes-clusters-in-vsphere.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in vSphere. +In [this section,](provision-kubernetes-clusters-in-vsphere.md) you'll learn how to use Rancher to install an [RKE2](https://docs.rke2.io/)/[K3s](https://docs.k3s.io/) Kubernetes cluster in vSphere. ## Provisioning Storage diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md index 2c32c5c3660..fb48a8c25fe 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -48,14 +48,13 @@ Rancher will discover and show resources created by `kubectl`. However, these re ## Authenticating Directly with a Downstream Cluster -This section intended to help you set up an alternative method to access an [RKE cluster.](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) +This section intended to help you set up an alternative method to access a [Rancher-launched cluster.](../../launch-kubernetes-with-rancher/launch-kubernetes-with-rancher.md) -This method is only available for RKE, RKE2, and K3s clusters that have the [authorized cluster endpoint](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled. When Rancher creates the cluster, it generates a kubeconfig file that includes additional kubectl context(s) for accessing your cluster. This additional context allows you to use kubectl to authenticate with the downstream cluster without authenticating through Rancher. For a longer explanation of how the authorized cluster endpoint works, refer to [this page](authorized-cluster-endpoint.md). +This method is only available RKE2 and K3s clusters that have the [authorized cluster endpoint](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled. When Rancher creates the cluster, it generates a kubeconfig file that includes additional kubectl context(s) for accessing your cluster. This additional context allows you to use kubectl to authenticate with the downstream cluster without authenticating through Rancher. For a longer explanation of how the authorized cluster endpoint works, refer to [this page](authorized-cluster-endpoint.md). On RKE2 and K3s clusters, you need to [manually enable](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) authorized cluster endpoints. -We recommend that as a best practice, you should set up this method to access your RKE, RKE2, and K3s clusters, so that just in case you can’t connect to Rancher, you can still access the cluster. - +We recommend that as a best practice, you should set up this method to access your RKE2 and K3s clusters, so that just in case you can’t connect to Rancher, you can still access the cluster. :::note Prerequisites: @@ -74,7 +73,7 @@ CURRENT NAME CLUSTER AUTHINFO N In this example, when you use `kubectl` with the first context, `my-cluster`, you will be authenticated through the Rancher server. -With the second context, `my-cluster-controlplane-1`, you would authenticate with the authorized cluster endpoint, communicating with an downstream RKE cluster directly. +With the second context, `my-cluster-controlplane-1`, you would authenticate with the authorized cluster endpoint, communicating with an downstream RKE/K3s cluster directly. We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md index 2ed1d943849..0bd0562745e 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -123,65 +123,6 @@ Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). ## Cleaning up Nodes - - -Before you run the following commands, first remove the node through the Rancher UI. - -To remove a node: - -1. Click **☰** and select **Cluster Management**. -1. In the table of clusters, click the name of the cluster the node belongs to. -1. In the first tab, click the checkbox next to the node's state. -1. Click **Delete**. - -If you remove the entire cluster instead of an individual node, or skip rermoving the node through the Rancher UI, follow these steps: - -1. [Remove](#docker-containers-images-and-volumes) the Docker containers from the node and [unmount](#mounts) any volumes. -1. Reboot the node. -1. [Remove](#directories-and-files) any remaining files. -1. Confirm that network interfaces and IP tables were properly cleaned after the reboot. If not, reboot one more time. - -### Windows Nodes - -To clean up a Windows node, run the script in `c:\etc\rancher`. This script deletes Kubernetes-generated resources and the execution binary. It also drops the firewall rules and network settings: - -``` -pushd c:\etc\rancher -.\cleanup.ps1 -popd -``` - -After you run this script, the node is reset and can be re-added to a Kubernetes cluster. - -### Docker Containers, Images, and Volumes - -:::caution - -Be careful when cleaning up Docker containers. The following command will remove *all* Docker containers, images, and volumes on the node, including non-Rancher related containers: - -::: - -``` -docker rm -f $(docker ps -qa) -docker rmi -f $(docker images -q) -docker volume rm $(docker volume ls -q) -``` - -### Mounts - -Kubernetes components and secrets leave behind the following mounts: - -* `/var/lib/kubelet` -* `/var/lib/rancher` -* Miscellaneous mounts in `/var/lib/kubelet/pods/` - -To unmount all mounts, run: - -``` -for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done -``` - - :::note @@ -248,54 +189,7 @@ Depending on the role you assigned to the node, certain directories may or may n ::: - - - -| Directories | -|------------------------------| -| `/etc/ceph` | -| `/etc/cni` | -| `/etc/kubernetes` | -| `/opt/cni` | -| `/opt/rke` | -| `/run/calico` | -| `/run/flannel` | -| `/run/secrets/kubernetes.io` | -| `/var/lib/calico` | -| `/var/lib/cni` | -| `/var/lib/etcd` | -| `/var/lib/kubelet` | -| `/var/lib/rancher/rke` | -| `/var/lib/weave` | -| `/var/log/containers` | -| `/var/log/kube-audit` | -| `/var/log/pods` | -| `/var/run/calico` | - -**To clean the directories:** - -```shell -rm -rf /etc/ceph \ - /etc/cni \ - /etc/kubernetes \ - /opt/cni \ - /opt/rke \ - /run/calico \ - /run/flannel \ - /run/secrets/kubernetes.io \ - /var/lib/calico \ - /var/lib/cni \ - /var/lib/etcd \ - /var/lib/kubelet \ - /var/lib/rancher/rke \ - /var/lib/weave \ - /var/log/containers \ - /var/log/kube-audit \ - /var/log/pods \ - /var/run/calico -``` - - + | Directories | diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md index 6f1010ff99c..259fcecbcf6 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -16,20 +16,6 @@ By default, Kubernetes clusters require certificates and Rancher launched Kubern Certificates can be rotated for the following services: - - - -- etcd -- kubelet (node certificate) -- kubelet (serving certificate, if [enabled](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)) -- kube-apiserver -- kube-proxy -- kube-scheduler -- kube-controller-manager - - - - - admin - api-server - controller-manager @@ -42,9 +28,6 @@ Certificates can be rotated for the following services: - kubelet - kube-proxy - - - :::note For users who didn't rotate their webhook certificates, and they have expired after one year, please see this [page](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md) for help. @@ -68,15 +51,4 @@ Rancher launched Kubernetes clusters have the ability to rotate the auto-generat ### Additional Notes - - - -Even though the RKE CLI can use custom certificates for the Kubernetes cluster components, Rancher currently doesn't allow the ability to upload these in Rancher launched Kubernetes clusters. - - - - -In RKE2, both etcd and control plane nodes are treated as the same `server` concept. As such, when rotating certificates of services specific to either of these components will result in certificates being rotated on both. The certificates will only change for the specified service, but you will see nodes for both components go into an updating state. You may also see worker only nodes go into an updating state. This is to restart the workers after a certificate change to ensure they get the latest client certs. - - - +In RKE2/K3s, both etcd and control plane nodes are treated as the same `server` concept. As such, when rotating certificates of services specific to either of these components will result in certificates being rotated on both. The certificates will only change for the specified service, but you will see nodes for both components go into an updating state. You may also see worker only nodes go into an updating state. This is to restart the workers after a certificate change to ensure they get the latest client certs. diff --git a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md index 5d56f2c60aa..59d1eed2fac 100644 --- a/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md +++ b/versioned_docs/version-2.12/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -6,39 +6,11 @@ title: Encryption Key Rotation -### RKE1 Encryption Key Rotation +:::note Important -1. Enable encryption key rotation with either of the following two options: +Encryption key rotation is enabled by default and cannot be disabled. - - Select the `Enabled` radio button in the Rancher UI under **Cluster Options > Advanced Options > Secrets Encryption**: - - ![Enable Encryption Key Rotation](/img/rke1-enable-secrets-encryption.png) - - - OR, apply the following YAML: - - ```yaml - rancher_kubernetes_engine_config: - services: - kube_api: - secrets_encryption_config: - enabled: true - ``` - -2. Rotate keys in the Rancher UI: - - 2.1. Click **☰ > Cluster Management**. - - 2.2. Select **⋮ > Rotate Encryption Keys** on the far right of the screen next to your chosen cluster: - - ![Encryption Key Rotation](/img/rke1-encryption-key.png) - - - -### RKE2 Encryption Key Rotation - -_**New in v2.6.7**_ - ->**Important:** Encryption key rotation is enabled by default and cannot be disabled. +::: To rotate keys in the Rancher UI: @@ -48,5 +20,4 @@ To rotate keys in the Rancher UI: ![Encryption Key Rotation](/img/rke2-encryption-key.png) - ->**Note:** For more information on RKE2 secrets encryption config, please see the [RKE2 docs](https://docs.rke2.io/security/secrets_encryption). \ No newline at end of file +>**Note:** For more information on RKE2 secrets encryption config, please see the [RKE2 docs](https://docs.rke2.io/security/secrets_encryption). diff --git a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md b/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md deleted file mode 100644 index 795e64cef29..00000000000 --- a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Roles-based Access Control ---- - - - - - -This section describes the permissions required to use the rancher-cis-benchmark App. - -The rancher-cis-benchmark is a cluster-admin only feature by default. - -However, the `rancher-cis-benchmark` chart installs these two default `ClusterRoles`: - -- cis-admin -- cis-view - -In Rancher, only cluster owners and global administrators have `cis-admin` access by default. - -Note: If you were using the `cis-edit` role added in Rancher v2.5 setup, it has now been removed since -Rancher v2.5.2 because it essentially is same as `cis-admin`. If you happen to create any clusterrolebindings -for `cis-edit`, please update them to use `cis-admin` ClusterRole instead. - -## Cluster-Admin Access - -Rancher CIS Scans is a cluster-admin only feature by default. -This means only the Rancher global admins, and the cluster’s cluster-owner can: - -- Install/Uninstall the rancher-cis-benchmark App -- See the navigation links for CIS Benchmark CRDs - ClusterScanBenchmarks, ClusterScanProfiles, ClusterScans -- List the default ClusterScanBenchmarks and ClusterScanProfiles -- Create/Edit/Delete new ClusterScanProfiles -- Create/Edit/Delete a new ClusterScan to run the CIS scan on the cluster -- View and Download the ClusterScanReport created after the ClusterScan is complete - - -## Summary of Default Permissions for Kubernetes Default Roles - -The rancher-cis-benchmark creates three `ClusterRoles` and adds the CIS Benchmark CRD access to the following default K8s `ClusterRoles`: - -| ClusterRole created by chart | Default K8s ClusterRole | Permissions given with Role -| ------------------------------| ---------------------------| ---------------------------| -| `cis-admin` | `admin`| Ability to CRUD clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR -| `cis-view` | `view `| Ability to List(R) clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR - - -By default only cluster-owner role will have ability to manage and use `rancher-cis-benchmark` feature. - -The other Rancher roles (cluster-member, project-owner, project-member) do not have any default permissions to manage and use rancher-cis-benchmark resources. - -But if a cluster-owner wants to delegate access to other users, they can do so by creating ClusterRoleBindings between these users and the above CIS ClusterRoles manually. -There is no automatic role aggregation supported for the `rancher-cis-benchmark` ClusterRoles. diff --git a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md b/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md deleted file mode 100644 index 3920a1588c5..00000000000 --- a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Skipped and Not Applicable Tests ---- - - - - - -This section lists the tests that are skipped in the permissive test profile for RKE. - -> All the tests that are skipped and not applicable on this page will be counted as Not Applicable in the v2.5 generated report. The skipped test count will only mention the user-defined skipped tests. This allows user-skipped tests to be distinguished from the tests that are skipped by default in the RKE permissive test profile. - -## CIS Benchmark v1.5 - -### CIS Benchmark v1.5 Skipped Tests - -| Number | Description | Reason for Skipping | -| ---------- | ------------- | --------- | -| 1.1.12 | Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) | A system service account is required for etcd data directory ownership. Refer to Rancher's hardening guide for more details on how to configure this ownership. | -| 1.2.6 | Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | -| 1.2.16 | Ensure that the admission control plugin PodSecurityPolicy is set (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 1.2.33 | Ensure that the --encryption-provider-config argument is set as appropriate (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | -| 1.2.34 | Ensure that encryption providers are appropriately configured (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | -| 4.2.6 | Ensure that the --protect-kernel-defaults argument is set to true (Automated) | System level configurations are required before provisioning the cluster in order for this argument to be set to true. | -| 4.2.10 | Ensure that the--tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | -| 5.1.5 | Ensure that default service accounts are not actively used. (Automated) | Kubernetes provides default service accounts to be used. | -| 5.2.2 | Minimize the admission of containers wishing to share the host process ID namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.2.3 | Minimize the admission of containers wishing to share the host IPC namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.2.4 | Minimize the admission of containers wishing to share the host network namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.2.5 | Minimize the admission of containers with allowPrivilegeEscalation (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | -| 5.3.2 | Ensure that all Namespaces have Network Policies defined (Automated) | Enabling Network Policies can prevent certain applications from communicating with each other. | -| 5.6.4 | The default namespace should not be used (Automated) | Kubernetes provides a default namespace. | - -### CIS Benchmark v1.5 Not Applicable Tests - -| Number | Description | Reason for being not applicable | -| ---------- | ------------- | --------- | -| 1.1.1 | Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | -| 1.1.2 | Ensure that the API server pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | -| 1.1.3 | Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.1.4 | Ensure that the controller manager pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.1.5 | Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.6 | Ensure that the scheduler pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.7 | Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | -| 1.1.8 | Ensure that the etcd pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | -| 1.1.13 | Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | -| 1.1.14 | Ensure that the admin.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | -| 1.1.15 | Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.16 | Ensure that the scheduler.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | -| 1.1.17 | Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.1.18 | Ensure that the controller-manager.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | -| 1.3.6 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | -| 4.1.1 | Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | -| 4.1.2 | Ensure that the kubelet service file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | -| 4.1.9 | Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | -| 4.1.10 | Ensure that the kubelet configuration file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | -| 4.2.12 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | \ No newline at end of file diff --git a/versioned_docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/versioned_docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md index 1f1d6335a21..997aaa48171 100644 --- a/versioned_docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md +++ b/versioned_docs/version-2.12/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -19,9 +19,7 @@ In order to deploy and run the adapter successfully, you need to ensure its vers | Rancher Version | Adapter Version | |-----------------|------------------| -| v2.11.2 | v106.0.0+up6.0.0 | -| v2.11.1 | v106.0.0+up6.0.0 | -| v2.11.0 | v106.0.0+up6.0.0 | +| v2.12.0 | 107.0.0+up7.0.0 | ### 1. Gain Access to the Local Cluster diff --git a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/cis-scans.md b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/compliance-scans.md similarity index 86% rename from versioned_docs/version-2.12/integrations-in-rancher/cis-scans/cis-scans.md rename to versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/compliance-scans.md index f170f997d66..a8b5ede68f0 100644 --- a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/cis-scans.md +++ b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/compliance-scans.md @@ -1,14 +1,14 @@ --- -title: CIS Scans +title: Compliance Scans --- - + -Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. The CIS scans can run on any Kubernetes cluster, including hosted Kubernetes providers such as EKS, AKS, and GKE. +Rancher can run a security scan to check whether a cluster is deployed according to security best practices as defined in Kubernetes security benchmarks, such as the ones provided by STIG, BSI or CIS. The Compliance scans can run on any Kubernetes cluster, including hosted Kubernetes providers such as EKS, AKS, and GKE. -The `rancher-cis-benchmark` app leverages kube-bench, an open-source tool from Aqua Security, to check clusters for CIS Kubernetes Benchmark compliance. Also, to generate a cluster-wide report, the application utilizes Sonobuoy for report aggregation. +The `rancher-compliance` app leverages kube-bench, an open-source tool from Aqua Security, to check the compliance of clusters against Kubernetes Benchmarks. Also, to generate a cluster-wide report, the application utilizes Sonobuoy for report aggregation. ## About the CIS Benchmark @@ -94,24 +94,22 @@ In order to pass the "Hardened" profile, you will need to follow the steps on th The default profile and the supported CIS benchmark version depends on the type of cluster that will be scanned: -The `rancher-cis-benchmark` supports the CIS 1.6 Benchmark version. +The `rancher-compliance` supports the CIS 1.9 Benchmark version. -- For RKE Kubernetes clusters, the RKE Permissive 1.6 profile is the default. +- For RKE Kubernetes clusters, the RKE Permissive 1.9 profile is the default. - EKS and GKE have their own CIS Benchmarks published by `kube-bench`. The corresponding test profiles are used by default for those clusters. -- For RKE2 Kubernetes clusters, the RKE2 Permissive 1.6 profile is the default. +- For RKE2 Kubernetes clusters, the RKE2 Permissive 1.9 profile is the default. - For cluster types other than RKE, RKE2, EKS and GKE, the Generic CIS 1.5 profile will be used by default. ## About Skipped and Not Applicable Tests -For a list of skipped and not applicable tests, refer to [this page](../../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md). - For now, only user-defined skipped tests are marked as skipped in the generated report. Any skipped tests that are defined as being skipped by one of the default profiles are marked as not applicable. ## Roles-based Access Control -For information about permissions, refer to [this page](rbac-for-cis-scans.md) +For information about permissions, refer to [this page](rbac-for-compliance-scans.md) ## Configuration @@ -119,4 +117,4 @@ For more information about configuring the custom resources for the scans, profi ## How-to Guides -Please refer to the [CIS Scan Guides](../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) to learn how to run CIS scans. +Please refer to the [Compliance Scan Guides](../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) to learn how to run Compliance scans. diff --git a/docs/integrations-in-rancher/cis-scans/configuration-reference.md b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/configuration-reference.md similarity index 63% rename from docs/integrations-in-rancher/cis-scans/configuration-reference.md rename to versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/configuration-reference.md index 3394bc2702b..4406a091acf 100644 --- a/docs/integrations-in-rancher/cis-scans/configuration-reference.md +++ b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/configuration-reference.md @@ -3,27 +3,27 @@ title: Configuration --- - + -This configuration reference is intended to help you manage the custom resources created by the `rancher-cis-benchmark` application. These resources are used for performing CIS scans on a cluster, skipping tests, setting the test profile that will be used during a scan, and other customization. +This configuration reference is intended to help you manage the custom resources created by the `rancher-compliance` application. These resources are used for performing compliance scans on a cluster, skipping tests, setting the test profile that will be used during a scan, and other customization. -To configure the custom resources, go to the **Cluster Dashboard** To configure the CIS scans, +To configure the custom resources, go to the **Cluster Dashboard** To configure the compliance scans, 1. In the upper left corner, click **☰ > Cluster Management**. -1. On the **Clusters** page, go to the cluster where you want to configure CIS scans and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark**. +1. On the **Clusters** page, go to the cluster where you want to configure compliance scans and click **Explore**. +1. In the left navigation bar, click **Compliance**. ## Scans -A scan is created to trigger a CIS scan on the cluster based on the defined profile. A report is created after the scan is completed. +A scan is created to trigger a compliance scan on the cluster based on the defined profile. A report is created after the scan is completed. When configuring a scan, you need to define the name of the scan profile that will be used with the `scanProfileName` directive. An example ClusterScan custom resource is below: ```yaml -apiVersion: cis.cattle.io/v1 +apiVersion: compliance.cattle.io/v1 kind: ClusterScan metadata: name: rke-cis @@ -33,11 +33,11 @@ spec: ## Profiles -A profile contains the configuration for the CIS scan, which includes the benchmark version to use and any specific tests to skip in that benchmark. +A profile contains the configuration for the compliance scan, which includes the benchmark version to use and any specific tests to skip in that benchmark. :::caution -By default, a few ClusterScanProfiles are installed as part of the `rancher-cis-benchmark` chart. If a user edits these default benchmarks or profiles, the next chart update will reset them back. So it is advisable for users to not edit the default ClusterScanProfiles. +By default, a few ClusterScanProfiles are installed as part of the `rancher-compliance` chart. If a user edits these default benchmarks or profiles, the next chart update will reset them back. So it is advisable for users to not edit the default ClusterScanProfiles. ::: @@ -50,12 +50,12 @@ When you create a new profile, you will also need to give it a name. An example `ClusterScanProfile` is below: ```yaml -apiVersion: cis.cattle.io/v1 +apiVersion: compliance.cattle.io/v1 kind: ClusterScanProfile metadata: annotations: meta.helm.sh/release-name: clusterscan-operator - meta.helm.sh/release-namespace: cis-operator-system + meta.helm.sh/release-namespace: compliance-operator-system labels: app.kubernetes.io/managed-by: Helm name: "" @@ -70,9 +70,9 @@ spec: A benchmark version is the name of benchmark to run using `kube-bench`, as well as the valid configuration parameters for that benchmark. -A `ClusterScanBenchmark` defines the CIS `BenchmarkVersion` name and test configurations. The `BenchmarkVersion` name is a parameter provided to the `kube-bench` tool. +A `ClusterScanBenchmark` defines the Compliance `BenchmarkVersion` name and test configurations. The `BenchmarkVersion` name is a parameter provided to the `kube-bench` tool. -By default, a few `BenchmarkVersion` names and test configurations are packaged as part of the CIS scan application. When this feature is enabled, these default BenchmarkVersions will be automatically installed and available for users to create a ClusterScanProfile. +By default, a few `BenchmarkVersion` names and test configurations are packaged as part of the Compliance scan application. When this feature is enabled, these default BenchmarkVersions will be automatically installed and available for users to create a ClusterScanProfile. :::caution @@ -89,12 +89,12 @@ A ClusterScanBenchmark consists of the fields: An example `ClusterScanBenchmark` is below: ```yaml -apiVersion: cis.cattle.io/v1 +apiVersion: compliance.cattle.io/v1 kind: ClusterScanBenchmark metadata: annotations: meta.helm.sh/release-name: clusterscan-operator - meta.helm.sh/release-namespace: cis-operator-system + meta.helm.sh/release-namespace: compliance-operator-system creationTimestamp: "2020-08-28T18:18:07Z" generation: 1 labels: @@ -106,4 +106,4 @@ metadata: spec: clusterProvider: "" minKubernetesVersion: 1.15.0 -``` \ No newline at end of file +``` diff --git a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/custom-benchmark.md b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/custom-benchmark.md similarity index 69% rename from versioned_docs/version-2.12/integrations-in-rancher/cis-scans/custom-benchmark.md rename to versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/custom-benchmark.md index 4ec353cc60b..fc4bf0e3b4b 100644 --- a/versioned_docs/version-2.12/integrations-in-rancher/cis-scans/custom-benchmark.md +++ b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/custom-benchmark.md @@ -3,19 +3,20 @@ title: Creating a Custom Benchmark Version for Running a Cluster Scan --- - + -Each Benchmark Version defines a set of test configuration files that define the CIS tests to be run by the kube-bench tool. -The `rancher-cis-benchmark` application installs a few default Benchmark Versions which are listed under CIS Benchmark application menu. +Each Benchmark Version defines a set of test configuration files that define the Compliance tests to be run by the kube-bench tool. +The `rancher-compliance` application installs a few default Benchmark Versions which are listed under Compliance application menu. -But there could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. -It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. +But in the following cases, a custom configuration or remediation may be required: -When a cluster scan is run, you need to select a Profile which points to a specific Benchmark Version. +- Non-standard file locations: When Kubernetes binaries, configuration or certificate paths deviate from upstream benchmark defaults. +Example: Unlike traditional Kubernetes, K3s bundles control plane components into a single binary. Therefore,` --anonymous-auth` flag presence and configuration should be verified in K3s' logs (`journalctl`), not via `kube-apiserver` process checks (`ps`). -Follow all the steps below to add a custom Benchmark Version and run a scan using it. +- Alternative risk mitigations: If a setup doesn't meet a check but has an equally effective compensating control with justification. Or simply is not concerned by the check requirement because of its design. +Example: By default, K3s embeds the api server within the k3s process. There is no API server pod specification file, so verifying the latter's file permissions is not required. ## 1. Prepare the Custom Benchmark Version ConfigMap @@ -46,7 +47,7 @@ To prepare a custom benchmark version ConfigMap, suppose we want to add a custom 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark > Benchmark Version**. +1. In the left navigation bar, click **Compliance > Benchmark Version**. 1. Click **Create**. 1. Enter the **Name** and a description for your custom benchmark version. 1. Choose the cluster provider that your benchmark version applies to. @@ -60,7 +61,7 @@ To run a scan using your custom benchmark version, you need to add a new Profile 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark > Profile**. +1. In the left navigation bar, click **Compliance > Profile**. 1. Click **Create**. 1. Provide a **Name** and description. In this example, we name it `foo-profile`. 1. Choose the Benchmark Version from the dropdown. @@ -74,7 +75,7 @@ To run a scan, 1. In the upper left corner, click **☰ > Cluster Management**. 1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. -1. In the left navigation bar, click **CIS Benchmark > Scan**. +1. In the left navigation bar, click **Compliance > Scan**. 1. Click **Create**. 1. Choose the new cluster scan profile. 1. Click **Create**. diff --git a/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans.md b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans.md new file mode 100644 index 00000000000..71348248436 --- /dev/null +++ b/versioned_docs/version-2.12/integrations-in-rancher/compliance-scans/rbac-for-compliance-scans.md @@ -0,0 +1,48 @@ +--- +title: Roles-based Access Control +--- + + + + + +This section describes the permissions required to use the rancher-compliance App. + +The rancher-compliance is a cluster-admin only feature by default. + +However, the `rancher-compliance` chart installs these two default `ClusterRoles`: + +- compliance-admin +- compliance-view + +In Rancher, only cluster owners and global administrators have `compliance-admin` access by default. + +## Cluster-Admin Access + +Rancher Compliance Scans is a cluster-admin only feature by default. +This means only the Rancher global admins, and the cluster’s cluster-owner can: + +- Install/Uninstall the rancher-compliance App +- See the navigation links for Compliance CRDs - ClusterScanBenchmarks, ClusterScanProfiles, ClusterScans +- List the default ClusterScanBenchmarks and ClusterScanProfiles +- Create/Edit/Delete new ClusterScanProfiles +- Create/Edit/Delete a new ClusterScan to run the Compliance scan on the cluster +- View and Download the ClusterScanReport created after the ClusterScan is complete + + +## Summary of Default Permissions for Kubernetes Default Roles + +The rancher-compliance creates three `ClusterRoles` and adds the Compliance CRD access to the following default K8s `ClusterRoles`: + +| ClusterRole created by chart | Default K8s ClusterRole | Permissions given with Role +| ------------------------------| ---------------------------| ---------------------------| +| `compliance-admin` | `admin`| Ability to CRUD clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR +| `compliance-view` | `view `| Ability to List(R) clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR + + +By default only cluster-owner role will have ability to manage and use `rancher-compliance` feature. + +The other Rancher roles (cluster-member, project-owner, project-member) do not have any default permissions to manage and use rancher-compliance resources. + +But if a cluster-owner wants to delegate access to other users, they can do so by creating ClusterRoleBindings between these users and the above Compliance ClusterRoles manually. +There is no automatic role aggregation supported for the `rancher-compliance` ClusterRoles. diff --git a/versioned_docs/version-2.12/integrations-in-rancher/harvester/overview.md b/versioned_docs/version-2.12/integrations-in-rancher/harvester/overview.md index edd54a6f557..ac94aa363b2 100644 --- a/versioned_docs/version-2.12/integrations-in-rancher/harvester/overview.md +++ b/versioned_docs/version-2.12/integrations-in-rancher/harvester/overview.md @@ -24,7 +24,7 @@ To navigate to the Harvester cluster, click **☰ > Virtualization Management**. ## Harvester Node Driver -The [Harvester node driver](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/) is generally available for RKE and RKE2 options in Rancher. The node driver is available whether or not the Harvester feature flag is enabled. Note that the node driver is off by default. Users may create RKE or RKE2 clusters on Harvester only from the Cluster Management page. +The [Harvester node driver](https://docs.harvesterhci.io/v1.5/rancher/node/node-driver/) is generally available for K3s and RKE2 options in Rancher. The node driver is available whether or not the Harvester feature flag is enabled. Note that the node driver is off by default. Users may create K3s or RKE2 clusters on Harvester only from the Cluster Management page. Harvester allows `.ISO` images to be uploaded and displayed through the Harvester UI, but this is not supported in the Rancher UI. This is because `.ISO` images usually require additional setup that interferes with a clean deployment (without requiring user intervention), and they are not typically used in cloud environments. @@ -32,11 +32,11 @@ See [Provisioning Drivers](../../how-to-guides/new-user-guides/authentication-pe ## Port Requirements -The port requirements for the Harvester cluster can be found [here](https://docs.harvesterhci.io/v1.1/install/requirements#networking). +The port requirements for the Harvester cluster can be found [here](https://docs.harvesterhci.io/v1.5/install/requirements#networking). In addition, other networking considerations are as follows: - Be sure to enable VLAN trunk ports of the physical switch for VM VLAN networks. -- Follow the networking setup guidance [here](https://docs.harvesterhci.io/v1.1/networking/index). +- Follow the networking setup guidance [here](https://docs.harvesterhci.io/v1.5/networking/index). -For other port requirements for other guest clusters, such as K3s and RKE1, please see [these docs](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters). +For other port requirements for other guest clusters, such as K3s and RKE2, please see [these docs](https://docs.harvesterhci.io/v1.5/install/requirements/#guest-clusters). diff --git a/versioned_docs/version-2.12/reference-guides/backup-restore-configuration/backup-configuration.md b/versioned_docs/version-2.12/reference-guides/backup-restore-configuration/backup-configuration.md index 054a6de3daa..2ece25be1a0 100644 --- a/versioned_docs/version-2.12/reference-guides/backup-restore-configuration/backup-configuration.md +++ b/versioned_docs/version-2.12/reference-guides/backup-restore-configuration/backup-configuration.md @@ -33,12 +33,6 @@ While you can create your own ResourceSets to back up custom applications, two R `rancher-resource-set-full` includes all essential secrets in the backup files to ensure Rancher continues running smoothly after a restore or migration. To avoid storing sensitive information in plain text, we strongly advise you to enable encryption with a strong key. -:::note Important: - -`rancher-resource-set` is also included by default with the `rancher-backup` operator. However, this ResourceSet is deprecated and is only being kept for backwards compatibility reasons. `rancher-resource-set` will be removed in Rancher v2.12. Please update your Backup custom resources to use either `rancher-resource-set-full` or `rancher-resource-set-basic`. - -::: - | YAML Directive Name | Description | | ---------------- | ---------------- | | `resourceSetName` | Provide the name of the ResourceSet to define which resources will be included in this backup. | diff --git a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md index c3c9b7a732d..e0b9eb6757f 100644 --- a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md +++ b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/disconnected-clusters.md @@ -16,4 +16,4 @@ While a managed cluster is disconnected from Rancher, management operations will - **Cleaning Up Disconnected Clusters**: Regularly remove clusters that will no longer reconnect to Rancher (e.g., clusters that have been decommissioned or destroyed). Keeping such clusters in the Rancher management system consumes unnecessary resources, which could impact Rancher's performance over time. -- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE/RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. +- **Certificate Rotation Considerations**: When designing processes that involve regularly shutting down clusters, whether connected to Rancher or not, take into account certificate rotation policies. For example, RKE2/K3s clusters may rotate certificates on startup if they exceeded their lifetime. diff --git a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index dc200ae62b1..7fda8ae4c35 100644 --- a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -30,9 +30,7 @@ Once you have created these _ClusterOutput_ objects, create a _ClusterFlow_ to c ### Kubernetes Components -_ClusterFlows_ have the ability to collect logs from all containers on all hosts in the Kubernetes cluster. This works well in cases where those containers are part of a Kubernetes pod; however, RKE containers exist outside of the scope of Kubernetes. - -Currently the logs from RKE containers are collected, but are not able to easily be filtered. This is because those logs do not contain information as to the source container (e.g. `etcd` or `kube-apiserver`). +_ClusterFlows_ have the ability to collect logs from all containers on all hosts in the Kubernetes cluster. This works well in cases where those containers are part of a Kubernetes pod. A future release of Rancher will include the source container name which will enable filtering of these component logs. Once that change is made, you will be able to customize a _ClusterFlow_ to retrieve **only** the Kubernetes component logs, and direct them to an appropriate output. diff --git a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md index 108689861dd..ffa2096211d 100644 --- a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md +++ b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -98,7 +98,7 @@ Monitoring the availability and performance of all your internal workloads is vi ## Security Monitoring -In addition to monitoring workloads to detect performance, availability or scalability problems, the cluster and the workloads running into it should also be monitored for potential security problems. A good starting point is to frequently run and alert on [CIS Scans](../../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) which check if the cluster is configured according to security best practices. +In addition to monitoring workloads to detect performance, availability or scalability problems, the cluster and the workloads running into it should also be monitored for potential security problems. A good starting point is to frequently run and alert on [Compliance Scans](../../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) which check if the cluster is configured according to security best practices. For the workloads, you can have a look at Kubernetes and Container security solutions like [NeuVector](https://www.suse.com/products/neuvector/), [Falco](https://falco.org/), [Aqua Kubernetes Security](https://www.aquasec.com/solutions/kubernetes-container-security/), [SysDig](https://sysdig.com/). diff --git a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index c8e12b81efe..877b5084bf9 100644 --- a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -54,9 +54,6 @@ Consider the following recommendations based on your needs: ### Make sure nodes are configured correctly for Kubernetes It's important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node, checking that all correct ports are opened, and deploying with ssd backed etcd. More details can be found in the [kubernetes docs](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin) and [etcd's performance op guide](https://etcd.io/docs/v3.5/op-guide/performance/). -### When using RKE: Back up the Statefile -RKE keeps record of the cluster state in a file called `cluster.rkestate`. This file is important for the recovery of a cluster and/or the continued maintenance of the cluster through RKE. Because this file contains certificate material, we strongly recommend encrypting this file before backing up. After each run of `rke up` you should backup the state file. - ### Run All Nodes in the Cluster in the Same Datacenter For best performance, run all three of your nodes in the same geographic datacenter. If you are running nodes in the cloud, such as AWS, run each node in a separate Availability Zone. For example, launch node 1 in us-west-2a, node 2 in us-west-2b, and node 3 in us-west-2c. diff --git a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md index a760bbebff1..87a74b259c2 100644 --- a/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md +++ b/versioned_docs/version-2.12/reference-guides/best-practices/rancher-server/tuning-and-best-practices-for-rancher-at-scale.md @@ -66,7 +66,7 @@ You should remove any remaining legacy apps that appear in the Cluster Manager U ### Using the Authorized Cluster Endpoint (ACE) -An [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) (ACE) provides access to the Kubernetes API of Rancher-provisioned RKE, RKE2, and K3s clusters. When enabled, the ACE adds a context to kubeconfig files generated for the cluster. The context uses a direct endpoint to the cluster, thereby bypassing Rancher. This reduces load on Rancher for cases where unmediated API access is acceptable or preferable. See [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) for more information and configuration instructions. +An [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) (ACE) provides access to the Kubernetes API of Rancher-provisioned RKE2 and K3s clusters. When enabled, the ACE adds a context to kubeconfig files generated for the cluster. The context uses a direct endpoint to the cluster, thereby bypassing Rancher. This reduces load on Rancher for cases where unmediated API access is acceptable or preferable. See [Authorized Cluster Endpoint](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) for more information and configuration instructions. ### Reducing Event Handler Executions diff --git a/versioned_docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md b/versioned_docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md index 8abd3377435..75bae23a493 100644 --- a/versioned_docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md +++ b/versioned_docs/version-2.12/reference-guides/cluster-configuration/cluster-configuration.md @@ -14,7 +14,6 @@ For information on editing cluster membership, go to [this page.](../../how-to-g The cluster configuration options depend on the type of Kubernetes cluster: -- [RKE Cluster Configuration](rancher-server-configuration/rke1-cluster-configuration.md) - [RKE2 Cluster Configuration](rancher-server-configuration/rke2-cluster-configuration.md) - [K3s Cluster Configuration](rancher-server-configuration/k3s-cluster-configuration.md) - [EKS Cluster Configuration](rancher-server-configuration/eks-cluster-configuration.md) diff --git a/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md b/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md index e0c92a52f0f..b10fe62447f 100644 --- a/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md +++ b/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/downstream-cluster-configuration.md @@ -6,4 +6,4 @@ title: Downstream Cluster Configuration -The following docs will discuss [node template configuration](node-template-configuration/node-template-configuration.md) and [machine configuration](machine-configuration/machine-configuration.md). \ No newline at end of file +The following docs will discuss [machine configuration](machine-configuration/machine-configuration.md). \ No newline at end of file diff --git a/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md b/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md new file mode 100644 index 00000000000..0960df5f441 --- /dev/null +++ b/versioned_docs/version-2.12/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce.md @@ -0,0 +1,86 @@ +--- +title: GCE Machine Configuration +--- + + + + + + +For more information about Google Cloud Platform (GCP) and the Google Compute Engine (GCE), refer to the official [GCP documentation](https://cloud.google.com/docs). + +### Zone + +The GCP Region and Zone that the VM will be deployed to. For example, `us-east1-b`. + +### Machine Image Project + +The image project that the desired image families belong to. + +### Machine Image Family + +The image family that the desired machine operating system belongs to. + +### Machine Image + +The operating system that will be installed onto the VM. + +### Disk Type + +The type of the disk attached to the VM. The available types may differ between regions. + +### Disk Size + +The size of the disk attached to the VM, in Gigabytes. + +### Machine Type + +The type of VM that will be deployed. Machine types determine the number of resources (vCPU, RAM, etc.) allocated for each node. + +### Network + +The VPC network that the VM will be created in. This value cannot be changed once the machine pool has been provisioned. + +### Subnet + +The VPC subnetwork tha the VM will be created in. This value cannot be changed once the machine pool has been provisioned. + +### Username + +A custom username set as the default user of the GCE VM. + +### External Address + +The desired external IP address for the GCE VM. + +### Scopes + +A list of OAuth2 scopes which allow the VM to access other GCP APIs. + +### Allow Internal Communication + +By default, a VPC firewall rule is automatically created to expose a fixed set of ports within the VPC to facilitate communication between cluster nodes. This behavior can be disabled on a per machine pool basis, when clicking the `Show Advanced` option and disabling the `Allow Internal Communication` checkbox. + +### Expose External ports + +A list of ports to be opened _externally_ to the wider internet. Open ports are defined at the machine pool level. Enabling this option will result in the automatic creation of a VPC firewall rule. This rule will be automatically deleted when the cluster or machine pool is deleted. + +### Network Tags + +Tags is a list of _network tags_, which can be used to associate preexisting Firewall Rules with all VMs within a machine pool. + +### Labels + +A comma seperated list of custom labels to be attached to all VMs within a given machine pool. Unlike Tags, Labels do not influence networking behavior and only serve to organize cloud resources. + +## Advanced Options + +When creating clusters via the Rancher UI some options are automatically configured for you. However, when creating machine config objects manually, you must ensure you properly configure the below fields. + +### external-firewall-rule-prefix + +A prefix that will be used when creating the firewall rule to expose ports publicly. Ideally, this should be a concatenation the machine pool name and the cluster name. This field must be set if the machine pool is configured to expose ports publicly, otherwise it can be omitted. + +### internal-firewall-rule-prefix + +A prefix that will be used when creating the internal firewall rule which allows for communication between nodes within the cluster. If this field is omitted, no internal firewall rule will be created. diff --git a/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md b/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md index 0e2aa590833..82be0707c8a 100644 --- a/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md +++ b/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rancher-server-configuration.md @@ -6,7 +6,6 @@ title: Rancher Server Configuration -- [RKE1 Cluster Configuration](rke1-cluster-configuration.md) - [RKE2 Cluster Configuration](rke2-cluster-configuration.md) - [K3s Cluster Configuration](k3s-cluster-configuration.md) - [EKS Cluster Configuration](eks-cluster-configuration.md) diff --git a/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md b/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md index a76cb30552d..44d7d22123b 100644 --- a/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md +++ b/versioned_docs/version-2.12/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md @@ -133,9 +133,9 @@ If the cloud provider you want to use is not listed as an option, you will need The default [pod security admission configuration template](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for the cluster. -##### Worker CIS Profile +##### Worker Compliance Profile -Select a [CIS benchmark](../../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) to validate the system configuration against. +Select a [compliance benchmark](../../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) to validate the system configuration against. ##### Project Network Isolation diff --git a/versioned_docs/version-2.12/reference-guides/kubernetes-concepts.md b/versioned_docs/version-2.12/reference-guides/kubernetes-concepts.md index e1e880e8cd4..c28d05394ce 100644 --- a/versioned_docs/version-2.12/reference-guides/kubernetes-concepts.md +++ b/versioned_docs/version-2.12/reference-guides/kubernetes-concepts.md @@ -10,13 +10,7 @@ This page explains concepts related to Kubernetes that are important for underst ## About Docker -Docker is the container packaging and runtime standard. Developers build container images from Dockerfiles and distribute container images from Docker registries. [Docker Hub](https://hub.docker.com) is the most popular public registry. Many organizations also set up private Docker registries. Docker is primarily used to manage containers on individual nodes. - -:::note - -Although Rancher 1.6 supported Docker Swarm clustering technology, it is no longer supported in Rancher 2.x due to the success of Kubernetes. - -::: +Docker is a container packaging and runtime standard. Developers can build container images from Dockerfiles and distribute container images from Docker registries. [Docker Hub](https://hub.docker.com) is a popular public registry. Many organizations also set up private Docker registries. Docker is primarily used to manage containers on individual nodes. ## About Kubernetes @@ -26,7 +20,7 @@ Kubernetes is the container cluster management standard. YAML files specify cont A cluster is a group of computers that work together as a single system. -A _Kubernetes Cluster_ is a cluster that uses the [Kubernetes container-orchestration system](https://kubernetes.io/) to deploy, maintain, and scale Docker containers, allowing your organization to automate application operations. +A _Kubernetes Cluster_ is a cluster that uses the [Kubernetes container-orchestration system](https://kubernetes.io/) to deploy, maintain, and scale containers, allowing your organization to automate application operations. ## Roles for Nodes in Kubernetes Clusters diff --git a/versioned_docs/version-2.12/reference-guides/monitoring-v2-configuration/receivers.md b/versioned_docs/version-2.12/reference-guides/monitoring-v2-configuration/receivers.md index b1237e3646b..16e3940e735 100644 --- a/versioned_docs/version-2.12/reference-guides/monitoring-v2-configuration/receivers.md +++ b/versioned_docs/version-2.12/reference-guides/monitoring-v2-configuration/receivers.md @@ -351,29 +351,29 @@ receivers: - service_key: 'database-integration-key' ``` -## Example Route Config for CIS Scan Alerts +## Example Route Config for Compliance Scan Alerts -While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. +While configuring the routes for `rancher-compliance` alerts, you can specify the matching using the key-value pair `job: rancher-compliance-scan`. -For example, the following example route configuration could be used with a Slack receiver named `test-cis`: +For example, the following example route configuration could be used with a Slack receiver named `test-compliance`: ```yaml spec: - receiver: test-cis + receiver: test-compliance group_by: # - string group_wait: 30s group_interval: 30s repeat_interval: 30s match: - job: rancher-cis-scan + job: rancher-compliance-scan # key: string match_re: {} # key: string ``` -For more information on enabling alerting for `rancher-cis-benchmark`, see [this section.](../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) +For more information on enabling alerting for `rancher-compliance-benchmark`, see [this section.](../../how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance.md) ## Trusted CA for Notifiers diff --git a/versioned_docs/version-2.12/reference-guides/rancher-cluster-tools.md b/versioned_docs/version-2.12/reference-guides/rancher-cluster-tools.md index ad46fbdd9d2..b6874436335 100644 --- a/versioned_docs/version-2.12/reference-guides/rancher-cluster-tools.md +++ b/versioned_docs/version-2.12/reference-guides/rancher-cluster-tools.md @@ -42,8 +42,8 @@ Rancher's integration with Istio was improved in Rancher v2.5. For more information, refer to the Istio documentation [here.](../integrations-in-rancher/istio/istio.md) -## CIS Scans +## Compliance Scans -Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. +Rancher can run a security scan to check whether a cluster is deployed according to security best practices as defined in Kubernetes security benchmarks, such as the ones provided by STIG, BSI or CIS. -For more information, refer to the CIS scan documentation [here.](../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) \ No newline at end of file +For more information, refer to the Compliance scan documentation [here.](../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) diff --git a/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md index b84375fbda5..99157d3bbba 100644 --- a/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -32,14 +32,6 @@ One option for the underlying Kubernetes cluster is to use K3s Kubernetes. K3s i ![Architecture of a K3s Kubernetes Cluster Running the Rancher Management Server](/img/k3s-server-storage.svg) -### RKE Kubernetes Cluster Installations - -In an RKE installation, the cluster data is replicated on each of three etcd nodes in the cluster, providing redundancy and data duplication in case one of the nodes fails. - -
Architecture of an RKE Kubernetes Cluster Running the Rancher Management Server
- -![Architecture of an RKE Kubernetes cluster running the Rancher management server](/img/rke-server-storage.svg) - ## Recommended Load Balancer Configuration for Kubernetes Installations We recommend the following configurations for the load balancer and Ingress controllers: @@ -61,7 +53,7 @@ For the best performance and greater security, we recommend a dedicated Kubernet ## Recommended Node Roles for Kubernetes Installations -The below recommendations apply when Rancher is installed on a K3s Kubernetes cluster or an RKE Kubernetes cluster. +The below recommendations apply when Rancher is installed on a K3s Kubernetes cluster. ### K3s Cluster Roles @@ -69,38 +61,6 @@ In K3s clusters, there are two types of nodes: server nodes and agent nodes. Bot For the cluster running the Rancher management server, we recommend using two server nodes. Agent nodes are not required. -### RKE Cluster Roles - -If Rancher is installed on an RKE Kubernetes cluster, the cluster should have three nodes, and each node should have all three Kubernetes roles: etcd, controlplane, and worker. - -### Contrasting RKE Cluster Architecture for Rancher Server and for Downstream Kubernetes Clusters - -Our recommendation for RKE node roles on the Rancher server cluster contrasts with our recommendations for the downstream user clusters that run your apps and services. - -Rancher uses RKE as a library when provisioning downstream Kubernetes clusters. Note: The capability to provision downstream K3s clusters will be added in a future version of Rancher. - -For downstream Kubernetes clusters, we recommend that each node in a user cluster should have a single role for stability and scalability. - -![Kubernetes Roles for Nodes in Rancher Server Cluster vs. User Clusters](/img/rancher-architecture-node-roles.svg) - -RKE only requires at least one node with each role and does not require nodes to be restricted to one role. However, for the clusters that run your apps, we recommend separate roles for each node so that workloads on worker nodes don't interfere with the Kubernetes master or cluster data as your services scale. - -We recommend that downstream user clusters should have at least: - -- **Three nodes with only the etcd role** to maintain a quorum if one node is lost, making the state of your cluster highly available -- **Two nodes with only the controlplane role** to make the master component highly available -- **One or more nodes with only the worker role** to run the Kubernetes node components, as well as the workloads for your apps and services - -With that said, it is safe to use all three roles on three nodes when setting up the Rancher server because: - -* It allows one `etcd` node failure. -* It maintains multiple instances of the master components by having multiple `controlplane` nodes. -* No other workloads than Rancher itself should be created on this cluster. - -Because no additional workloads will be deployed on the Rancher server cluster, in most cases it is not necessary to use the same architecture that we recommend for the scalability and reliability of downstream clusters. - -For more best practices for downstream clusters, refer to the [production checklist](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/checklist-for-production-ready-clusters.md) or our [best practices guide.](../best-practices/best-practices.md) - ## Architecture for an Authorized Cluster Endpoint (ACE) If you are using an [authorized cluster endpoint (ACE),](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) we recommend creating an FQDN pointing to a load balancer which balances traffic across your nodes with the `controlplane` role. diff --git a/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index e3dd9cb475e..f42e97da652 100644 --- a/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/versioned_docs/version-2.12/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -41,7 +41,7 @@ There is one cluster controller and one cluster agent for each downstream cluste - Watches for resource changes in the downstream cluster - Brings the current state of the downstream cluster to the desired state - Configures access control policies to clusters and projects -- Provisions clusters by calling the required Docker machine drivers and Kubernetes engines, such as RKE and GKE +- Provisions clusters by calling the required Docker machine drivers and Kubernetes engines, such as GKE By default, to enable Rancher to communicate with a downstream cluster, the cluster controller connects to the cluster agent. If the cluster agent is not available, the cluster controller can connect to a [node agent](#3-node-agents) instead. @@ -62,7 +62,7 @@ The `cattle-node-agent` is deployed using a [DaemonSet](https://kubernetes.io/do An authorized cluster endpoint (ACE) allows users to connect to the Kubernetes API server of a downstream cluster without having to route their requests through the Rancher authentication proxy. -> ACE is available on RKE, RKE2, and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. +> ACE is available on RKE2 and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. There are two main reasons why a user might need the authorized cluster endpoint: @@ -178,11 +178,8 @@ If you see an error related to "impersonation" in the UI, pay close attention to The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster: -- `rancher-cluster.yml`: The RKE cluster configuration file. -- `kube_config_rancher-cluster.yml`: The Kubeconfig file for the cluster, this file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. -- `rancher-cluster.rkestate`: The Kubernetes cluster state file. This file contains credentials for full access to the cluster. Note: This state file is only created when using RKE v0.2.0 or higher. - -> **Note:** The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. +- `config.yaml`: The RKE2 and K3s cluster configuration file. +- `rke2.yaml` or `k3s.yaml`: The Kubeconfig file for your RKE2 or K3s cluster. This file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. For more information on connecting to a cluster without the Rancher authentication proxy and other configuration options, refer to the [kubeconfig file](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) documentation. @@ -194,13 +191,7 @@ The tools that Rancher uses to provision downstream user clusters depends on the Rancher can dynamically provision nodes in a provider such as Amazon EC2, DigitalOcean, Azure, or vSphere, then install Kubernetes on them. -Rancher provisions this type of cluster using [RKE](https://github.com/rancher/rke) and [docker-machine.](https://github.com/rancher/machine) - -### Rancher Launched Kubernetes for Custom Nodes - -When setting up this type of cluster, Rancher installs Kubernetes on existing nodes, which creates a custom cluster. - -Rancher provisions this type of cluster using [RKE.](https://github.com/rancher/rke) +Rancher provisions this type of cluster using [docker-machine.](https://github.com/rancher/machine) ### Hosted Kubernetes Providers diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md deleted file mode 100644 index ca021c8dc53..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/hardening-guides.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Self-Assessment and Hardening Guides for Rancher ---- - - - - - -Rancher provides specific security hardening guides for each supported Rancher version's Kubernetes distributions. - -## Rancher Kubernetes Distributions - -Rancher uses the following Kubernetes distributions: - -- [**RKE**](https://rancher.com/docs/rke/latest/en/), Rancher Kubernetes Engine, is a CNCF-certified Kubernetes distribution that runs entirely within Docker containers. -- [**RKE2**](https://docs.rke2.io/) is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. -- [**K3s**](https://docs.k3s.io/) is a fully conformant, lightweight Kubernetes distribution. It is easy to install, with half the memory requirement of upstream Kubernetes, all in a binary of less than 100 MB. - -To harden a Kubernetes cluster that's running a distribution other than those listed, refer to your Kubernetes provider docs. - -## Hardening Guides and Benchmark Versions - -Each self-assessment guide is accompanied by a hardening guide. These guides were tested alongside the listed Rancher releases. Each self-assessment guides was tested on a specific Kubernetes version and CIS benchmark version. If a CIS benchmark has not been validated for your Kubernetes version, you can use the existing guides until a guide for your version is added. - -### RKE Guides - -| Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | -|--------------------|-----------------------|-----------------------|------------------| -| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke1-hardening-guide/rke1-hardening-guide.md) | - -### RKE2 Guides - -| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | -|------|--------------------|-----------------------|-----------------------|------------------| -| Rancher provisioned RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke2-hardening-guide/rke2-hardening-guide.md) | -| Standalone RKE2 | Kubernetes v1.27-v1.32 | CIS v1.9 | [Link](https://docs.rke2.io/security/cis_self_assessment19) | [Link](https://docs.rke2.io/security/hardening_guide) | - -### K3s Guides - -| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | -|------|--------------------|-----------------------|-----------------------|------------------| -| Rancher provisioned K3s cluster | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](k3s-hardening-guide/k3s-hardening-guide.md) | -| Standalone K3s | Kubernetes v1.26 up to v1.29 | CIS v1.8 | [Link](https://docs.k3s.io/security/self-assessment-1.8) | [Link](https://docs.k3s.io/security/hardening-guide) | - -## Rancher with SELinux - -[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a kernel module that adds extra access controls and security tools to Linux. Historically used by government agencies, SELinux is now industry-standard. SELinux is enabled by default on RHEL and CentOS. - -To use Rancher with SELinux, we recommend [installing](../selinux-rpm/about-rancher-selinux.md) the `rancher-selinux` RPM. - diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide.md deleted file mode 100644 index 5878e17f31b..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide.md +++ /dev/null @@ -1,744 +0,0 @@ ---- -title: K3s Hardening Guides ---- - - - - - -This document provides prescriptive guidance for how to harden a K3s cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. - -:::note -This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. -::: - -This hardening guide is intended to be used for K3s clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | - -:::note -In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter isn't required anymore, and was removed by CIS. -::: - -For more details on how to evaluate a hardened K3s cluster against the official CIS benchmark, refer to the K3s self-assessment guides for specific Kubernetes and CIS benchmark versions. - -K3s passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: - -1. K3s does not modify the host operating system. Any host-level modifications need to be done manually. -2. Certain CIS policy controls for `NetworkPolicies` and `PodSecurityStandards` (`PodSecurityPolicies` on v1.24 and older) restrict cluster functionality. - You must opt into having K3s configure these policies. Add the appropriate options to your command-line flags or configuration file (enable admission plugins), and manually apply the appropriate policies. - See further for more details. - -The first section (1.1) of the CIS Benchmark primarily focuses on pod manifest permissions and ownership. Since everything in the distribution is packaged in a single binary, this section does not apply to the core components of K3s. - -## Host-level Requirements - -### Ensure `protect-kernel-defaults` is set - - - - -The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. - - - - -This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. - -The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. - -```yaml -spec: - rkeConfig: - machineSelectorConfig: - - config: - protect-kernel-defaults: true -``` - - - - -### Set kernel parameters - -The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: - -```ini -vm.panic_on_oom=0 -vm.overcommit_memory=1 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. - -This configuration needs to be done before setting the kubelet flag, otherwise K3s will fail to start. - -## Kubernetes Runtime Requirements - -The CIS Benchmark runtime requirements center around pod security (via PSA), network policies and API Server auditing logs. - -By default, K3s does not include any pod security or network policies. However, K3s ships with a controller that enforces any network policies you create. By default, K3s enables both the `PodSecurity` and `NodeRestriction` admission controllers, among others. - -### Pod Security - - - - -K3s v1.25 and newer support [Pod Security admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) for controlling pod security. - -You can specify the PSA configuration by setting the `defaultPodSecurityAdmissionConfigurationTemplateName` field in the cluster configuration in Rancher: - -```yaml -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted -``` - -The `rancher-restricted` template is provided by Rancher to enforce the highly-restrictive Kubernetes upstream [`Restricted`](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile with best practices for pod hardening. - - - - -K3s v1.24 and older support [Pod Security Policy (PSP)](https://github.com/kubernetes/website/blob/release-1.24/content/en/docs/concepts/security/pod-security-policy.md) for controlling pod security. - -You can enable PSPs by passing the following flags in the cluster configuration in Rancher: - -```yaml -spec: - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount -``` - -This maintains the `NodeRestriction` plugin and enables the `PodSecurityPolicy`. - -Once you enable PSPs, you can apply a policy to satisfy the necessary controls described in section 5.2 of the CIS Benchmark. - -:::note -These are manual checks in the CIS Benchmark. The CIS scan flags the results as `warning`, because manual inspection is necessary by the cluster operator. -::: - -Here is an example of a compliant PSP: - -```yaml ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: restricted-psp -spec: - privileged: false # CIS - 5.2.1 - allowPrivilegeEscalation: false # CIS - 5.2.5 - requiredDropCapabilities: # CIS - 5.2.7/8/9 - - ALL - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false # CIS - 5.2.4 - hostIPC: false # CIS - 5.2.3 - hostPID: false # CIS - 5.2.2 - runAsUser: - rule: 'MustRunAsNonRoot' # CIS - 5.2.6 - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - readOnlyRootFilesystem: false -``` - -For the example PSP to be effective, we need to create a `ClusterRole` and a `ClusterRoleBinding`. We also need to include a "system unrestricted policy" for system-level pods that require additional privileges, and an additional policy that allows the necessary sysctls for full functionality of ServiceLB. - -```yaml ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: restricted-psp -spec: - privileged: false - allowPrivilegeEscalation: false - requiredDropCapabilities: - - ALL - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - rule: 'MustRunAsNonRoot' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - readOnlyRootFilesystem: false ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: system-unrestricted-psp - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' -spec: - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - fsGroup: - rule: RunAsAny - hostIPC: true - hostNetwork: true - hostPID: true - hostPorts: - - max: 65535 - min: 0 - privileged: true - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - '*' ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: svclb-psp - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' -spec: - allowPrivilegeEscalation: false - allowedCapabilities: - - NET_ADMIN - allowedUnsafeSysctls: - - net.ipv4.ip_forward - - net.ipv6.conf.all.forwarding - fsGroup: - rule: RunAsAny - hostPorts: - - max: 65535 - min: 0 - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:restricted-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - restricted-psp ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:system-unrestricted-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svclb-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - svclb-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svc-local-path-provisioner-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svc-coredns-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:svc-cis-operator-psp -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - system-unrestricted-psp - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: default:restricted-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:restricted-psp -subjects: -- kind: Group - name: system:authenticated - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system-unrestricted-node-psp-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:system-unrestricted-psp -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system-unrestricted-svc-acct-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:system-unrestricted-psp -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svclb-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svclb-psp -subjects: -- kind: ServiceAccount - name: svclb ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svc-local-path-provisioner-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svc-local-path-provisioner-psp -subjects: -- kind: ServiceAccount - name: local-path-provisioner-service-account ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svc-coredns-psp-rolebinding - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svc-coredns-psp -subjects: -- kind: ServiceAccount - name: coredns ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: svc-cis-operator-psp-rolebinding - namespace: cis-operator-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:svc-cis-operator-psp -subjects: -- kind: ServiceAccount - name: cis-operator-serviceaccount -``` - -The policies presented above can be placed in a file named `policy.yaml` in the `/var/lib/rancher/k3s/server/manifests` directory. Both the policy file and the its directory hierarchy must be created before starting K3s. A restrictive access permission is recommended to avoid leaking potential sensitive information. - -```shell -sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests -``` - -:::note -The critical Kubernetes additions such as CNI, DNS, and Ingress are run as pods in the `kube-system` namespace. Therefore, this namespace has a less restrictive policy, so that these components can run properly. -::: - - - - -### Network Policies - -CIS requires that all namespaces apply a network policy that reasonably limits traffic into namespaces and pods. - -:::note -This is a manual check in the CIS Benchmark. The CIS scan flags the result as a `warning`, because manual inspection is necessary by the cluster operator. -::: - -The network policies can be placed in the `policy.yaml` file in `/var/lib/rancher/k3s/server/manifests` directory. If the directory was not created as part of the PSP (as described above), it must be created first. - -```shell -sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests -``` - -Here is an example of a compliant network policy: - -```yaml ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: kube-system -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: kube-system ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: default -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: default ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: intra-namespace - namespace: kube-public -spec: - podSelector: {} - ingress: - - from: - - namespaceSelector: - matchLabels: - name: kube-public -``` - -The active restrictions block DNS unless purposely allowed. Below is a network policy that allows DNS-related traffic: - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-network-dns-policy - namespace: -spec: - ingress: - - ports: - - port: 53 - protocol: TCP - - port: 53 - protocol: UDP - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Ingress -``` - -The metrics-server and Traefik ingress controller are blocked by default if network policies are not created to allow access. - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-metrics-server - namespace: kube-system -spec: - podSelector: - matchLabels: - k8s-app: metrics-server - ingress: - - {} - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-svclbtraefik-ingress - namespace: kube-system -spec: - podSelector: - matchLabels: - svccontroller.k3s.cattle.io/svcname: traefik - ingress: - - {} - policyTypes: - - Ingress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-all-traefik-v121-ingress - namespace: kube-system -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: traefik - ingress: - - {} - policyTypes: - - Ingress -``` - -:::note -You must manage network policies as normal for any additional namespaces you create. -::: - -### API Server audit configuration - -CIS requirements 1.2.22 to 1.2.25 are related to configuring audit logs for the API Server. K3s does not create by default the log directory and audit policy, as auditing requirements are specific to each user's policies and environment. - -If you need a log directory, it must be created before you start K3s. We recommend a restrictive access permission to avoid leaking sensitive information. - -```bash -sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/logs -``` - -The following is a starter audit policy to log request metadata. This policy should be written to a file named `audit.yaml` in the `/var/lib/rancher/k3s/server` directory. Detailed information about policy configuration for the API server can be found in the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). - -```yaml ---- -apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: Metadata -``` - -Further configurations are also needed to pass CIS checks. These are not configured by default in K3s, because they vary based on your environment and needs: - -- Ensure that the `--audit-log-path` argument is set. -- Ensure that the `--audit-log-maxage` argument is set to 30 or as appropriate. -- Ensure that the `--audit-log-maxbackup` argument is set to 10 or as appropriate. -- Ensure that the `--audit-log-maxsize` argument is set to 100 or as appropriate. - -Combined, to enable and configure audit logs, add the following lines to the K3s cluster configuration file in Rancher: - -```yaml -spec: - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 - - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 - - audit-log-maxage=30 # CIS 1.2.19 - - audit-log-maxbackup=10 # CIS 1.2.20 - - audit-log-maxsize=100 # CIS 1.2.21 -``` - -### Controller Manager Requirements - -CIS requirement 1.3.1 checks for garbage collection settings in the Controller Manager. Garbage collection is important to ensure sufficient resource availability and avoid degraded performance and availability. Based on your system resources and tests, choose an appropriate threshold value to activate garbage collection. - -This can be remediated by setting the following configuration in the K3s cluster file in Rancher. The value below is only an example. The appropriate threshold value is specific to each user's environment. - -```yaml -spec: - rkeConfig: - machineGlobalConfig: - kube-controller-manager-arg: - - terminated-pod-gc-threshold=10 # CIS 1.3.1 -``` - -### Configure `default` Service Account - -Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. - -For CIS requirement 5.1.5 the `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. - -This can be remediated by updating the `automountServiceAccountToken` field to `false` for the `default` service account in each namespace. - -For `default` service accounts in the built-in namespaces (`kube-system`, `kube-public`, `kube-node-lease`, and `default)`, K3s does not automatically do this. - -Save the following configuration to a file called `account_update.yaml`. - -```yaml ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -Create a bash script file called `account_update.sh`. Be sure to `chmod +x account_update.sh` so the script has execute permissions. - -```shell -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -Run the script every time a new service account is added to your cluster. - -## Reference Hardened K3s Template Configuration - -The following reference template configuration is used in Rancher to create a hardened K3s custom cluster based on each CIS control in this guide. This reference does not include other required **cluster configuration** directives, which vary based on your environment. - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted - enableNetworkPolicy: true - kubernetesVersion: # Define K3s version - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - enable-admission-plugins=NodeRestriction,ServiceAccount # CIS 1.2.15, 1.2.13 - - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 - - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 - - audit-log-maxage=30 # CIS 1.2.19 - - audit-log-maxbackup=10 # CIS 1.2.20 - - audit-log-maxsize=100 # CIS 1.2.21 - - request-timeout=300s # CIS 1.2.22 - - service-account-lookup=true # CIS 1.2.24 - kube-controller-manager-arg: - - terminated-pod-gc-threshold=10 # CIS 1.3.1 - secrets-encryption: true - machineSelectorConfig: - - config: - kubelet-arg: - - make-iptables-util-chains=true # CIS 4.2.7 -``` - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - enableNetworkPolicy: true - kubernetesVersion: # Define K3s version - rkeConfig: - machineGlobalConfig: - kube-apiserver-arg: - - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount # CIS 1.2.15, 5.2, 1.2.13 - - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 - - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 - - audit-log-maxage=30 # CIS 1.2.19 - - audit-log-maxbackup=10 # CIS 1.2.20 - - audit-log-maxsize=100 # CIS 1.2.21 - - request-timeout=300s # CIS 1.2.22 - - service-account-lookup=true # CIS 1.2.24 - kube-controller-manager-arg: - - terminated-pod-gc-threshold=10 # CIS 1.3.1 - secrets-encryption: true - machineSelectorConfig: - - config: - kubelet-arg: - - make-iptables-util-chains=true # CIS 4.2.7 - protect-kernel-defaults: true # CIS 4.2.6 -``` - - - - -## Conclusion - -If you have followed this guide, your K3s custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our K3s self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index fe6b3ef299c..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,3215 +0,0 @@ ---- -title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - -This document is a companion to the [K3s Hardening Guide](k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. - -This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -This document is for Rancher operators, security teams, auditors and decision makers. - -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). - -## Testing Methodology - -Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. - -Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. - -These are the possible results for each control: - -- **Pass** - The K3s cluster passes the audit outlined in the benchmark. -- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. -- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. - -This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. - -:::note - -This guide only covers `automated` (previously called `scored`) tests. - -::: - -### Controls - - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable. - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable. - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable. - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable. - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable. - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable. - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /etc/kubernetes/manifests/etcd.yaml -Not Applicable. - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /etc/kubernetes/manifests/etcd.yaml -Not Applicable. - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 -Not Applicable. - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root -Not Applicable. - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 1.1.11 -``` - -**Expected Result**: - -```console -'700' is equal to '700' -``` - -**Returned Value**: - -```console -700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** Not Applicable - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd -Not Applicable. - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 scheduler - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root scheduler - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 controllermanager - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root controllermanager - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/server/cred/controller.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /var/lib/rancher/k3s/server/tls - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/server/tls -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /etc/kubernetes/pki/*.crt - -**Audit:** - -```bash -stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /etc/kubernetes/pki/*.key - -**Audit:** - -```bash -stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= -Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' is present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... -Permissive - Enabling Pod Security Policy can cause applications to unexpectedly fail. - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log -Permissive. - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 -Permissive. - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 -Permissive. - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 -Permissive. - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s -Permissive. - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 1.2.29 -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= -Permissive - Enabling encryption changes how data can be recovered as data is encrypted. - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. -Permissive - Enabling encryption changes how data can be recovered as data is encrypted. - -### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true -Not Applicable. - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep containerd | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' -``` - -**Expected Result**: - -```console -'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.1 -``` - -**Expected Result**: - -```console -'cert-file' is present AND 'key-file' is present -``` - -**Returned Value**: - -```console -cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key -``` - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and set the below parameter. ---client-cert-auth="true" - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.2 -``` - -**Expected Result**: - -```console -'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -client-cert-auth: true -``` - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.3 -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.4 -``` - -**Expected Result**: - -```console -'cert-file' is present AND 'key-file' is present -``` - -**Returned Value**: - -```console -cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key -``` - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and set the below parameter. ---peer-client-cert-auth=true - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.5 -``` - -**Expected Result**: - -```console -'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -client-cert-auth: true -``` - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.6 -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit Script:** `check_for_k3s_etcd.sh` - -```bash -#!/bin/bash - -# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) -# before it checks the requirement -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - - -if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then - case $1 in - "1.1.11") - echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; - "1.2.29") - echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; - "2.1") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.2") - echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.3") - echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.4") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; - "2.5") - echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; - "2.6") - echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; - "2.7") - echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; - esac -else -# If another database is running, return whatever is required to pass the scan - case $1 in - "1.1.11") - echo "700";; - "1.2.29") - echo "--etcd-certfile AND --etcd-keyfile";; - "2.1") - echo "cert-file AND key-file";; - "2.2") - echo "--client-cert-auth=true";; - "2.3") - echo "false";; - "2.4") - echo "peer-cert-file AND peer-key-file";; - "2.5") - echo "--client-cert-auth=true";; - "2.6") - echo "--peer-auto-tls=false";; - "2.7") - echo "--trusted-ca-file";; - esac -fi - -``` - -**Audit Execution:** - -```bash -./check_for_k3s_etcd.sh 2.7 -``` - -**Expected Result**: - -```console -'trusted-ca-file' is present -``` - -**Returned Value**: - -```console -trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Manual) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - All configuration is passed in as arguments at container run time. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable. - All configuration is passed in as arguments at container run time. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -**Returned Value**: - -```console -600 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit:** - -```bash -stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -**Returned Value**: - -```console -644 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/kubelet/config.yaml - -### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/kubelet/config.yaml -Not Applicable. -All configuration is passed in as arguments at container run time. - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** warn - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' -``` - -**Expected Result**: - -```console -'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present -``` - -**Returned Value**: - -```console -Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable. - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--event-qps' is present OR '--event-qps' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** Not Applicable - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--rotate-certificates' is present OR '--rotate-certificates' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** pass - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable. - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/kubelet/config.yaml -``` - -**Expected Result**: - -```console -'RotateKubeletServerCertificate' is present OR 'RotateKubeletServerCertificate' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** warn - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC containerd -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--pod-max-pids' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** pass - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -**Audit Script:** `check_for_default_sa.sh` - -```bash -#!/bin/bash - -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - -count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) -if [[ ${count_sa} -gt 0 ]]; then - echo "false" - exit -fi - -for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") -do - for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') - do - read kind name <<<$(IFS=","; echo $result) - resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) - if [[ ${resource_count} -gt 0 ]]; then - echo "false" - exit - fi - done -done - - -echo "true" - -``` - -**Audit Execution:** - -```bash -./check_for_default_sa.sh -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Automated) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Automated) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md deleted file mode 100644 index afa5dc0fef1..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide.md +++ /dev/null @@ -1,513 +0,0 @@ ---- -title: RKE Hardening Guides ---- - - - - - - - -This document provides prescriptive guidance for how to harden an RKE cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. - -:::note -This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. -::: - -This hardening guide is intended to be used for RKE clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | - -:::note -- In Benchmark v1.24 and later, check id `4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated)` might fail, as `/etc/kubernetes/ssl/kube-ca.pem` is set to 644 by default. -- In Benchmark v1.7, the `--protect-kernel-defaults` (`4.2.6`) parameter isn't required anymore, and was removed by CIS. -::: - -For more details on how to evaluate a hardened RKE cluster against the official CIS benchmark, refer to the RKE self-assessment guides for specific Kubernetes and CIS benchmark versions. - -## Host-level requirements - -### Configure Kernel Runtime Parameters - -The following `sysctl` configuration is recommended for all nodes types in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: - -```ini -vm.overcommit_memory=1 -vm.panic_on_oom=0 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -Run `sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. - -### Configure `etcd` user and group - -A user account and group for the **etcd** service is required to be set up before installing RKE. - -#### Create `etcd` user and group - -To create the **etcd** user and group run the following console commands. -The commands below use `52034` for **uid** and **gid** for example purposes. -Any valid unused **uid** or **gid** could also be used in lieu of `52034`. - -```bash -groupadd --gid 52034 etcd -useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin -``` - -When deploying RKE through its cluster configuration `config.yml` file, update the `uid` and `gid` of the `etcd` user: - -```yaml -services: - etcd: - gid: 52034 - uid: 52034 -``` - -## Kubernetes runtime requirements - -### Configure `default` Service Account - -#### Set `automountServiceAccountToken` to `false` for `default` service accounts - -Kubernetes provides a default service account which is used by cluster workloads where no specific service account is assigned to the pod. -Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. -The default service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. - -For each namespace including `default` and `kube-system` on a standard RKE install, the `default` service account must include this value: - -```yaml -automountServiceAccountToken: false -``` - -Save the following configuration to a file called `account_update.yaml`. - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -Create a bash script file called `account_update.sh`. -Be sure to `chmod +x account_update.sh` so the script has execute permissions. - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. - -### Configure Network Policy - -#### Ensure that all Namespaces have Network Policies defined - -Running different applications on the same Kubernetes cluster creates a risk of one compromised application attacking a neighboring application. Network segmentation is important to ensure that containers can communicate only with those they are supposed to. A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. - -Network Policies are namespace scoped. When a network policy is introduced to a given namespace, all traffic not allowed by the policy is denied. However, if there are no network policies in a namespace all traffic will be allowed into and out of the pods in that namespace. To enforce network policies, a container network interface (CNI) plugin must be enabled. This guide uses [Canal](https://github.com/projectcalico/canal) to provide the policy enforcement. Additional information about CNI providers can be found [here](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/). - -Once a CNI provider is enabled on a cluster a default network policy can be applied. For reference purposes a **permissive** example is provided below. If you want to allow all traffic to all pods in a namespace (even if policies are added that cause some pods to be treated as “isolated”), you can create a policy that explicitly allows all traffic in that namespace. Save the following configuration as `default-allow-all.yaml`. Additional [documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) about network policies can be found on the Kubernetes site. - -:::caution -This network policy is just an example and is not recommended for production use. -::: - -```yaml ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-allow-all -spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress -``` - -Create a bash script file called `apply_networkPolicy_to_all_ns.sh`. Be sure to `chmod +x apply_networkPolicy_to_all_ns.sh` so the script has execute permissions. - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - kubectl apply -f default-allow-all.yaml -n ${namespace} -done -``` - -Execute this script to apply the `default-allow-all.yaml` configuration with the **permissive** `NetworkPolicy` to all namespaces. - -## Known Limitations - -- Rancher **exec shell** and **view logs** for pods are **not** functional in a hardened setup when only a public IP is provided when registering custom nodes. This functionality requires a private IP to be provided when registering the custom nodes. - -## Reference Hardened RKE `cluster.yml` Configuration - -The reference `cluster.yml` is used by the RKE CLI that provides the configuration needed to achieve a hardened installation of RKE. RKE [documentation](https://rancher.com/docs/rke/latest/en/installation/) provides additional details about the configuration items. This reference `cluster.yml` does not include the required `nodes` directive which will vary depending on your environment. Documentation for node configuration in RKE can be found [here](https://rancher.com/docs/rke/latest/en/config-options/nodes/). - -The example `cluster.yml` configuration file contains an Admission Configuration policy in the `services.kube-api.admission_configuration` field. This [sample](../../psa-restricted-exemptions.md) policy contains the namespace exemptions necessary for an imported RKE cluster to run properly in Rancher, similar to Rancher's pre-defined [`rancher-restricted`](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) policy. - -If you prefer to use RKE's default `restricted` policy, then leave the `services.kube-api.admission_configuration` field empty and set `services.pod_security_configuration` to `restricted`. See [the RKE docs](https://rke.docs.rancher.com/config-options/services/pod-security-admission) for more information. - - - - -:::note -If you intend to import an RKE cluster into Rancher, please consult the [documentation](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. -::: - -```yaml -# If you intend to deploy Kubernetes in an air-gapped environment, -# please consult the documentation on how to configure custom RKE images. -nodes: [] -kubernetes_version: # Define RKE version -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - # Leave `pod_security_configuration` out if you are setting a - # custom policy in `admission_configuration`. Otherwise set - # it to `restricted` to use RKE's pre-defined restricted policy, - # and remove everything inside `admission_configuration` field. - # - # pod_security_configuration: restricted - # - admission_configuration: - apiVersion: apiserver.config.k8s.io/v1 - kind: AdmissionConfiguration - plugins: - - name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1 - kind: PodSecurityConfiguration - defaults: - enforce: "restricted" - enforce-version: "latest" - audit: "restricted" - audit-version: "latest" - warn: "restricted" - warn-version: "latest" - exemptions: - usernames: [] - runtimeClasses: [] - namespaces: [calico-apiserver, - calico-system, - cattle-alerting, - cattle-csp-adapter-system, - cattle-elemental-system, - cattle-epinio-system, - cattle-externalip-system, - cattle-fleet-local-system, - cattle-fleet-system, - cattle-gatekeeper-system, - cattle-global-data, - cattle-global-nt, - cattle-impersonation-system, - cattle-istio, - cattle-istio-system, - cattle-logging, - cattle-logging-system, - cattle-monitoring-system, - cattle-neuvector-system, - cattle-prometheus, - cattle-provisioning-capi-system, - cattle-resources-system, - cattle-sriov-system, - cattle-system, - cattle-ui-plugin-system, - cattle-windows-gmsa-system, - cert-manager, - cis-operator-system, - fleet-default, - ingress-nginx, - istio-system, - kube-node-lease, - kube-public, - kube-system, - longhorn-system, - rancher-alerting-drivers, - security-scan, - tigera-operator] - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - generate_serving_certificate: true -addons: | - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -```yaml -# If you intend to deploy Kubernetes in an air-gapped environment, -# please consult the documentation on how to configure custom RKE images. -nodes: [] -kubernetes_version: # Define RKE version -services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - secrets_encryption_config: - enabled: true - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - generate_serving_certificate: true -addons: | - # Upstream Kubernetes restricted PSP policy - # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: restricted-noroot - spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - requiredDropCapabilities: - - ALL - # Allow core volume types. - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. - - 'csi' - - 'persistentVolumeClaim' - - 'ephemeral' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - # Require the container to run without root privileges. - rule: 'MustRunAsNonRoot' - seLinux: - # This policy assumes the nodes are using AppArmor rather than SELinux. - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - # Forbid adding the root group. - - min: 1 - max: 65535 - readOnlyRootFilesystem: false - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: psp:restricted-noroot - rules: - - apiGroups: - - extensions - resourceNames: - - restricted-noroot - resources: - - podsecuritypolicies - verbs: - - use - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: psp:restricted-noroot - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:restricted-noroot - subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:authenticated - --- - apiVersion: networking.k8s.io/v1 - kind: NetworkPolicy - metadata: - name: default-allow-all - spec: - podSelector: {} - ingress: - - {} - egress: - - {} - policyTypes: - - Ingress - - Egress - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - automountServiceAccountToken: false -``` - - - - -## Reference Hardened RKE Cluster Template Configuration - -The reference RKE cluster template provides the minimum required configuration to achieve a hardened installation of Kubernetes. RKE templates are used to provision Kubernetes and define Rancher settings. Follow the Rancher [documentation](../../../../getting-started/installation-and-upgrade/installation-and-upgrade.md) for additional information about installing RKE and its template details. - - - - -```yaml -# -# Cluster Config -# -default_pod_security_admission_configuration_template_name: rancher-restricted -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # Define cluster name - -# -# Rancher Config -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # Define RKE version - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: false - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -```yaml -# -# Cluster Config -# -default_pod_security_policy_template_id: restricted-noroot -enable_network_policy: true -local_cluster_auth_endpoint: - enabled: true -name: # Define cluster name - -# -# Rancher Config -# -rancher_kubernetes_engine_config: - addon_job_timeout: 45 - authentication: - strategy: x509|webhook - kubernetes_version: # Define RKE version - services: - etcd: - uid: 52034 - gid: 52034 - kube-api: - audit_log: - enabled: true - event_rate_limit: - enabled: true - pod_security_policy: true - secrets_encryption_config: - enabled: true - kube-controller: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - kubelet: - extra_args: - feature-gates: RotateKubeletServerCertificate=true - protect-kernel-defaults: true - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - generate_serving_certificate: true - scheduler: - extra_args: - tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -``` - - - - -## Conclusion - -If you have followed this guide, your RKE custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index ac002a20369..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,2865 +0,0 @@ ---- -title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - - - -This document is a companion to the [RKE Hardening Guide](rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. - - -This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. - -This document is for Rancher operators, security teams, auditors and decision makers. - -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). - -## Testing Methodology - -Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. - -Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. - -:::note - -This guide only covers `automated` (previously called `scored`) tests. - -::: - -### Controls - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /etc/kubernetes/manifests/etcd.yaml -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. -All configuration is passed in as arguments at container run time. - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a -``` - -**Expected Result**: - -```console -'permissions' is present -``` - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root - -**Audit:** - -```bash -ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit:** - -```bash -stat -c %a /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'700' is equal to '700' -``` - -**Returned Value**: - -```console -700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd - -**Audit:** - -```bash -stat -c %U:%G /node/var/lib/etcd -``` - -**Expected Result**: - -```console -'etcd:etcd' is present -``` - -**Returned Value**: - -```console -etcd:etcd -``` - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf -Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root scheduler -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. -All configuration is passed in as arguments at container run time. - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root controllermanager -Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. -All configuration is passed in as arguments at container run time. - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /etc/kubernetes/pki/ - -**Audit Script:** `check_files_owner_in_dir.sh` - -```bash -#!/usr/bin/env bash - -# This script is used to ensure the owner is set to root:root for -# the given directory and all the files in it -# -# inputs: -# $1 = /full/path/to/directory -# -# outputs: -# true/false - -INPUT_DIR=$1 - -if [[ "${INPUT_DIR}" == "" ]]; then - echo "false" - exit -fi - -if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then - echo "false" - exit -fi - -statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) -while read -r statInfoLine; do - f=$(echo ${statInfoLine} | cut -d' ' -f1) - p=$(echo ${statInfoLine} | cut -d' ' -f2) - - if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then - if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then - echo "false" - exit - fi - else - if [[ "$p" != "root:root" ]]; then - echo "false" - exit - fi - fi -done <<< "${statInfoLines}" - - -echo "true" -exit - -``` - -**Audit Execution:** - -```bash -./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + - -**Audit:** - -```bash -find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= -When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'AlwaysPullImages' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-path' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxage' is greater or equal to 30 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxbackup' is greater or equal to 10 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxsize' is greater or equal to 100 -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--encryption-provider-config' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. - -**Audit:** - -```bash -ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi -``` - -**Expected Result**: - -```console -'provider' is present -``` - -### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true -Cluster provisioned by RKE handles certificate rotation directly through RKE. - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is present OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--cert-file' is present AND '--key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---client-cert-auth="true" - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-cert-file' is present AND '--peer-key-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and set the below parameter. ---peer-client-cert-auth=true - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--peer-client-cert-auth' is equal to 'true' -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'--trusted-ca-file' is present -``` - -**Returned Value**: - -```console -etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Automated) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. -All configuration is passed in as arguments at container run time. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. - All configuration is passed in as arguments at container run time. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit:** - -```bash -stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit:** - -```bash -stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/kubelet/config.yaml -Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. -All configuration is passed in as arguments at container run time. - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--event-qps' is greater or equal to 0 OR '--event-qps' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--rotate-certificates' is present OR '--rotate-certificates' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' -``` - -**Expected Result**: - -```console -'--pod-max-pids' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** pass - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -**Audit Script:** `check_for_default_sa.sh` - -```bash -#!/bin/bash - -set -eE - -handle_error() { - echo "false" -} - -trap 'handle_error' ERR - -count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) -if [[ ${count_sa} -gt 0 ]]; then - echo "false" - exit -fi - -for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") -do - for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') - do - read kind name <<<$(IFS=","; echo $result) - resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) - if [[ ${resource_count} -gt 0 ]]; then - echo "false" - exit - fi - done -done - - -echo "true" - -``` - -**Audit Execution:** - -```bash -./check_for_default_sa.sh -``` - -**Expected Result**: - -```console -'true' is equal to 'true' -``` - -**Returned Value**: - -```console -true -``` - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Manual) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md deleted file mode 100644 index bf79c65bd58..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide.md +++ /dev/null @@ -1,274 +0,0 @@ ---- -title: RKE2 Hardening Guides ---- - - - - - -This document provides prescriptive guidance for how to harden an RKE2 cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. - -:::note -This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. -::: - -This hardening guide is intended to be used for RKE2 clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|------------------------------| -| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | -| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | - -:::note -- In Benchmark v1.24 and later, some check ids might fail due to new file permission requirements (600 instead of 644). Impacted check ids: `1.1.1`, `1.1.3`, `1.1.5`, `1.1.7`, `1.1.13`, `1.1.15`, `1.1.17`, `4.1.3`, `4.1.5` and `4.1.9`. - - In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter is not required anymore, and was removed by CIS. -::: - -For more details on how to evaluate a hardened RKE2 cluster against the official CIS benchmark, refer to the RKE2 self-assessment guides for specific Kubernetes and CIS benchmark versions. - -RKE2 passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: - -1. RKE2 will not modify the host operating system. Therefore, you, the operator, must make a few host-level modifications. -2. Certain CIS controls for Network Policies and Pod Security Standards (or Pod Security Policies (PSP) on RKE2 versions prior to v1.25) will restrict the functionality of the cluster. You must opt into having RKE2 configure these for you. To help ensure these requirements are met, RKE2 can be started with the profile flag set to `cis-1.23` for v1.25 and newer or `cis-1.6` for v1.24 and older. - -## Host-level requirements - -There are two areas of host-level requirements: kernel parameters and etcd process/directory configuration. These are outlined in this section. - -### Ensure `protect-kernel-defaults` is set - - - - -The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. - - - - -This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. - -The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. - -```yaml -spec: - rkeConfig: - machineSelectorConfig: - - config: - protect-kernel-defaults: true -``` - - - - -### Set kernel parameters - -The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: - -```ini -vm.panic_on_oom=0 -vm.overcommit_memory=1 -kernel.panic=10 -kernel.panic_on_oops=1 -``` - -Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. - -### Ensure etcd is configured properly - -The CIS Benchmark requires that the etcd data directory be owned by the `etcd` user and group. This implicitly requires the etcd process run as the host-level `etcd` user. To achieve this, RKE2 takes several steps when started with a valid `cis-1.xx` profile: - -1. Check that the `etcd` user and group exists on the host. If they don't, exit with an error. -2. Create etcd's data directory with `etcd` as the user and group owner. -3. Ensure the etcd process is ran as the `etcd` user and group by setting the etcd static pod's `SecurityContext` appropriately. - -To meet the above requirements, you must: - -#### Create the etcd user - -On some Linux distributions, the `useradd` command will not create a group. The `-U` flag is included below to account for that. This flag tells `useradd` to create a group with the same name as the user. - -```bash -sudo useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U -``` - -## Kubernetes runtime requirements - -The runtime requirements to pass the CIS Benchmark are centered around pod security, network policies and kernel parameters. Most of this is automatically handled by RKE2 when using a valid `cis-1.xx` profile, but some additional operator intervention is required. These are outlined in this section. - -### PodSecurity - -RKE2 always runs with some amount of pod security. - - - - -On v1.25 and newer, [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) are used for pod security. - -Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.7-hardened` available in Rancher. - -```yaml -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.23 -``` - -When both the `defaultPodSecurityAdmissionConfigurationTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: - -1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. -2. Applies network policies that allow the cluster to pass associated controls. -3. Configures the Pod Security Admission Controller with the PSA configuration template `rancher-restricted`, to enforce restricted mode in all namespaces, except the ones in the template's exemption list. - These namespaces are exempted to allow system pods to run without restrictions, which is required for proper operation of the cluster. - -:::note -If you intend to import an RKE cluster into Rancher, please consult the [documentation](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. -::: - - - - - -On Kubernetes v1.24 and older, the `PodSecurityPolicy` admission controller is always enabled. - -Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. - -:::note -In the following example the profile is set to `cis-1.6` which is the value defined in the upstream RKE2, but the cluster is actually configured to pass the CIS v1.23 hardened profile -::: - -```yaml -spec: - defaultPodSecurityPolicyTemplateName: restricted-noroot - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.6 -``` - - -When both the `defaultPodSecurityPolicyTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: - -1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. -2. Applies network policies that allow the cluster to pass associated controls. -3. Configures runtime pod security policies that allow the cluster to pass associated controls. - - - - -:::note -The Kubernetes control plane components and critical additions such as CNI, DNS, and Ingress are ran as pods in the `kube-system` namespace. Therefore, this namespace will have a policy that is less restrictive so that these components can run properly. -::: - -### NetworkPolicies - -When ran with a valid `cis-1.xx` profile, RKE2 will put `NetworkPolicies` in place that passes the CIS Benchmark for Kubernetes' built-in namespaces. These namespaces are: `kube-system`, `kube-public`, `kube-node-lease`, and `default`. - -The `NetworkPolicy` used will only allow pods within the same namespace to talk to each other. The notable exception to this is that it allows DNS requests to be resolved. - -:::note -Operators must manage network policies as normal for additional namespaces that are created. -::: - -### Configure `default` service account - -**Set `automountServiceAccountToken` to `false` for `default` service accounts** - -Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. The `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. - -For each namespace including `default` and `kube-system` on a standard RKE2 install, the `default` service account must include this value: - -```yaml -automountServiceAccountToken: false -``` - -For namespaces created by the cluster operator, the following script and configuration file can be used to configure the `default` service account. - -The configuration bellow must be saved to a file called `account_update.yaml`. - -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default -automountServiceAccountToken: false -``` - -Create a bash script file called `account_update.sh`. Be sure to `sudo chmod +x account_update.sh` so the script has execute permissions. - -```bash -#!/bin/bash -e - -for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do - echo -n "Patching namespace $namespace - " - kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" -done -``` - -Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. - -### API Server audit configuration - -CIS requirements 1.2.19 to 1.2.22 are related to configuring audit logs for the API Server. When RKE2 is started with the `profile` flag set, it will automatically configure hardened `--audit-log-` parameters in the API Server to pass those CIS checks. - -RKE2's default audit policy is configured to not log requests in the API Server. This is done to allow cluster operators flexibility to customize an audit policy that suits their auditing requirements and needs, as these are specific to each users' environment and policies. - -A default audit policy is created by RKE2 when started with the `profile` flag set. The policy is defined in `/etc/rancher/rke2/audit-policy.yaml`. - -```yaml -apiVersion: audit.k8s.io/v1 -kind: Policy -metadata: - creationTimestamp: null -rules: -- level: None -``` - -## Reference Hardened RKE2 Template Configuration - -The reference template configuration is used in Rancher to create a hardened RKE2 custom cluster. This reference does not include other required **cluster configuration** directives which will vary depending on your environment. - - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted - kubernetesVersion: # Define RKE2 version - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.23 -``` - - - - -```yaml -apiVersion: provisioning.cattle.io/v1 -kind: Cluster -metadata: - name: # Define cluster name -spec: - defaultPodSecurityPolicyTemplateName: restricted-noroot - kubernetesVersion: # Define RKE2 version - rkeConfig: - machineSelectorConfig: - - config: - profile: cis-1.6 - protect-kernel-defaults: true -``` - - - - -## Conclusion - -If you have followed this guide, your RKE2 custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE2 self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md deleted file mode 100644 index c93773a62a7..00000000000 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md +++ /dev/null @@ -1,2967 +0,0 @@ ---- -title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 ---- - - - - - -This document is a companion to the [RKE2 Hardening Guide](rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. - -This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: - -| Rancher Version | CIS Benchmark Version | Kubernetes Version | -|-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | - -This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. - -This document is for Rancher operators, security teams, auditors and decision makers. - -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). - -## Testing Methodology - -RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. - -Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. - -:::note - -This guide only covers `automated` (previously called `scored`) tests. - -::: - -### Controls - -## 1.1 Control Plane Node Configuration Files -### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the -control plane node. -For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then find /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml -name '*etcd*' | xargs stat -c permissions=%a; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 - -**Audit:** - -```bash -ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=644 -``` - -### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root - -**Audit:** - -```bash -ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root root:root -``` - -### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) - - -**Result:** pass - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). For example, -chmod 700 /var/lib/etcd - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd -``` - -**Expected Result**: - -```console -permissions has permissions 700, expected 700 or more restrictive -``` - -**Returned Value**: - -```console -permissions=700 -``` - -### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) - - -**Result:** Not Applicable - -**Remediation:** -On the etcd server node, get the etcd data directory, passed as an argument --data-dir, -from the command 'ps -ef | grep etcd'. -Run the below command (based on the etcd data directory found above). -For example, chown etcd:etcd /var/lib/etcd - -### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chmod 600 /etc/kubernetes/admin.conf - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, chown root:root /etc/kubernetes/admin.conf - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'600' is present -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod 600 /var/lib/rancher/rke2/server/cred/controller.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'600' is present -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chown -R root:root /etc/kubernetes/pki/ - -**Audit:** - -```bash -stat -c %U:%G /var/lib/rancher/rke2/server/tls -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /var/lib/rancher/rke2/server/tls/*.crt - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 -``` - -### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the control plane node. -For example, -chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key - -**Audit:** - -```bash -stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 -``` - -## 1.2 API Server -### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---anonymous-auth=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and configure alternate mechanisms for authentication. Then, -edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and remove the --token-auth-file= parameter. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--token-auth-file' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and remove the `DenyServiceExternalIPs` -from enabled admission plugins. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the -apiserver and kubelets. Then, edit API server pod specification file -/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the -kubelet client certificate and key parameters as below. ---kubelet-client-certificate= ---kubelet-client-key= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and setup the TLS connection between -the apiserver and kubelets. Then, edit the API server pod specification file -/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the ---kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. ---kubelet-certificate-authority= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--kubelet-certificate-authority' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. -One such example could be as below. ---authorization-mode=RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes Node. ---authorization-mode=Node,RBAC - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'Node' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, -for example `--authorization-mode=Node,RBAC`. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--authorization-mode' has 'RBAC' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and set the desired limits in a configuration file. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -and set the below parameters. ---enable-admission-plugins=...,EventRateLimit,... ---admission-control-config-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'EventRateLimit' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a -value that does not include AlwaysAdmit. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -AlwaysPullImages. ---enable-admission-plugins=...,AlwaysPullImages,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'AlwaysPullImages' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) - - -**Result:** warn - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to include -SecurityContextDeny, unless PodSecurityPolicy is already in place. ---enable-admission-plugins=...,SecurityContextDeny,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the documentation and create ServiceAccount objects as per your environment. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and ensure that the --disable-admission-plugins parameter is set to a -value that does not include ServiceAccount. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1018768 2419 99 16:17 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz -``` - -### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --disable-admission-plugins parameter to -ensure it does not include NamespaceLifecycle. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --enable-admission-plugins parameter to a -value that includes NodeRestriction. ---enable-admission-plugins=...,NodeRestriction,... - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--enable-admission-plugins' has 'NodeRestriction' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and either remove the --secure-port parameter or -set it to a different (non-zero) desired port. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--secure-port' is greater than 0 OR '--secure-port' is not present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.17 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-path parameter to a suitable path and -file where you would like audit logs to be written, for example, ---audit-log-path=/var/log/apiserver/audit.log - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-path' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxage parameter to 30 -or as an appropriate number of days, for example, ---audit-log-maxage=30 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxage' is greater or equal to 30 -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate -value. For example, ---audit-log-maxbackup=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-log-maxbackup' is greater or equal to 10 -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. -For example, to set it as 100 MB, --audit-log-maxsize=100 - -### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -and set the below parameter as appropriate and if needed. -For example, --request-timeout=300s - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--request-timeout' is not present OR '--request-timeout' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---service-account-lookup=true -Alternatively, you can delete the --service-account-lookup parameter from this file so -that the default takes effect. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-lookup' is not present OR '--service-account-lookup' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --service-account-key-file parameter -to the public key file for service accounts. For example, ---service-account-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-key-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate and key file parameters. ---etcd-certfile= ---etcd-keyfile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-certfile' is present AND '--etcd-keyfile' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the TLS certificate and private key file parameters. ---tls-cert-file= ---tls-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection on the apiserver. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the client certificate authority file. ---client-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the etcd certificate authority file parameter. ---etcd-cafile= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--etcd-cafile' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml -on the control plane node and set the --encryption-provider-config parameter to the path of that file. -For example, --encryption-provider-config= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--encryption-provider-config' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and configure a EncryptionConfig file. -In this file, choose aescbc, kms or secretbox as the encryption provider. - -**Audit:** - -```bash -ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi -``` - -**Expected Result**: - -```console -'provider' is present -``` - -### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml -on the control plane node and set the below parameter. ---tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, -TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, -TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, -TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 -Not Applicable. - -## 1.3 Controller Manager -### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, -for example, --terminated-pod-gc-threshold=10 - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--terminated-pod-gc-threshold' is present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.2 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node to set the below parameter. ---use-service-account-credentials=true - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--use-service-account-credentials' is not equal to 'false' -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --service-account-private-key-file parameter -to the private key file for service accounts. ---service-account-private-key-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--service-account-private-key-file' is present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. ---root-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--root-ca-file' is present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. ---feature-gates=RotateKubeletServerCertificate=true - -### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-controller-manager | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -## 1.4 Scheduler -### 1.4.1 Ensure that the --profiling argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file -on the control plane node and set the below parameter. ---profiling=false - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--profiling' is equal to 'false' -``` - -**Returned Value**: - -```console -root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 -``` - -### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) - - -**Result:** pass - -**Remediation:** -Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml -on the control plane node and ensure the correct value for the --bind-address parameter - -**Audit:** - -```bash -/bin/ps -ef | grep kube-scheduler | grep -v grep -``` - -**Expected Result**: - -```console -'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present -``` - -**Returned Value**: - -```console -root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 -``` - -## 2 Etcd Node Configuration -### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the etcd service documentation and configure TLS encryption. -Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml -on the master node and set the below parameters. ---cert-file= ---key-file= -Not Applicable. - -### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and set the below parameter. ---client-cert-auth="true" -Not Applicable. - -### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and either remove the --auto-tls parameter or set it to false. - --auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ -``` - -### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Follow the etcd service documentation and configure peer TLS encryption as appropriate -for your etcd cluster. -Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the -master node and set the below parameters. ---peer-client-file= ---peer-key-file= -Not Applicable. - -### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and set the below parameter. ---peer-client-cert-auth=true -Not Applicable. - -### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) - - -**Result:** pass - -**Remediation:** -Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master -node and either remove the --peer-auto-tls parameter or set it to false. ---peer-auto-tls=false - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Expected Result**: - -```console -'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ -``` - -### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) - - -**Result:** pass - -**Remediation:** -[Manual test] -Follow the etcd documentation and create a dedicated certificate authority setup for the -etcd service. -Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the -master node and set the below parameter. ---trusted-ca-file= - -**Audit:** - -```bash -/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep -``` - -**Audit Config:** - -```bash -cat /var/lib/rancher/rke2/server/db/etcd/config -``` - -**Expected Result**: - -```console -'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' -``` - -**Returned Value**: - -```console -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ -``` - -## 3.1 Authentication and Authorization -### 3.1.1 Client certificate authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be -implemented in place of client certificates. - -### 3.1.2 Service account token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of service account tokens. - -### 3.1.3 Bootstrap token authentication should not be used for users (Manual) - - -**Result:** warn - -**Remediation:** -Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented -in place of bootstrap tokens. - -## 3.2 Logging -### 3.2.1 Ensure that a minimal audit policy is created (Automated) - - -**Result:** pass - -**Remediation:** -Create an audit policy file for your cluster. - -**Audit:** - -```bash -/bin/ps -ef | grep kube-apiserver | grep -v grep -``` - -**Expected Result**: - -```console -'--audit-policy-file' is present -``` - -**Returned Value**: - -```console -root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true -``` - -### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) - - -**Result:** warn - -**Remediation:** -Review the audit policy provided for the cluster and ensure that it covers -at least the following areas, -- Access to Secrets managed by the cluster. Care should be taken to only - log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in - order to avoid risk of logging sensitive data. -- Modification of Pod and Deployment objects. -- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. -For most requests, minimally logging at the Metadata level is recommended -(the most basic level of logging). - -## 4.1 Worker Node Configuration Files -### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not Applicable. - -### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) - - -**Result:** Not Applicable - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -Not applicable. - -### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) - - -**Result:** warn - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the below command (based on the file location on your system) on the each worker node. -For example, -chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the file permissions of the ---client-ca-file chmod 600 - -**Audit Script:** `check_cafile_permissions.sh` - -```bash -#!/usr/bin/env bash - -CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') -CAFILE=/node$CAFILE -if test -z $CAFILE; then CAFILE=$kubeletcafile; fi -if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi - -``` - -**Audit Execution:** - -```bash -./check_cafile_permissions.sh -``` - -**Expected Result**: - -```console -permissions has permissions 600, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=600 -``` - -### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) - - -**Result:** pass - -**Remediation:** -Run the following command to modify the ownership of the --client-ca-file. -chown root:root - -**Audit Script:** `check_cafile_ownership.sh` - -```bash -#!/usr/bin/env bash - -CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') -CAFILE=/node$CAFILE -if test -z $CAFILE; then CAFILE=$kubeletcafile; fi -if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi - -``` - -**Audit Execution:** - -```bash -./check_cafile_ownership.sh -``` - -**Expected Result**: - -```console -'root:root' is equal to 'root:root' -``` - -**Returned Value**: - -```console -root:root -``` - -### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) - - -**Result:** fail - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -permissions has permissions 644, expected 600 or more restrictive -``` - -**Returned Value**: - -```console -permissions=644 -``` - -### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) - - -**Result:** pass - -**Remediation:** -Run the following command (using the config file location identified in the Audit step) -chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig - -**Audit:** - -```bash -/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' -``` - -**Expected Result**: - -```console -'root:root' is present -``` - -**Returned Value**: - -```console -root:root -``` - -## 4.2 Kubelet -### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to -`false`. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -`--anonymous-auth=false` -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--anonymous-auth' is equal to 'false' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If -using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---authorization-mode=Webhook -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--authorization-mode' does not have 'AlwaysAllow' -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to -the location of the client CA file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_AUTHZ_ARGS variable. ---client-ca-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--client-ca-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---read-only-port=0 -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--read-only-port' is equal to '0' OR '--read-only-port' is not present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a -value other than 0. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. ---streaming-connection-idle-timeout=5m -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove the --make-iptables-util-chains argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) - - -**Result:** Not Applicable - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and remove the --hostname-override argument from the -KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service -Not Applicable. - -### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.eventRecordQPS}' is present OR '{.eventRecordQPS}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to set `tlsCertFile` to the location -of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` -to the location of the corresponding private key file. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the below parameters in KUBELET_CERTIFICATE_ARGS variable. ---tls-cert-file= ---tls-private-key-file= -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'--tls-cert-file' is present AND '--tls-private-key-file' is present -``` - -**Returned Value**: - -```console -UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key -``` - -### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) - - -**Result:** pass - -**Remediation:** -If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or -remove it altogether to use the default value. -If using command line arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS -variable. -Based on your system, restart the kubelet service. For example, -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) - - -**Result:** pass - -**Remediation:** -Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. ---feature-gates=RotateKubeletServerCertificate=true -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) - - -**Result:** warn - -**Remediation:** -If using a Kubelet config file, edit the file to set `TLSCipherSuites` to -TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -or to a subset of these values. -If using executable arguments, edit the kubelet service file -/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and -set the --tls-cipher-suites parameter as follows, or to a subset of these values. ---tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 -Based on your system, restart the kubelet service. For example: -systemctl daemon-reload -systemctl restart kubelet.service - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{range .tlsCipherSuites[:]}{}{','}{end}' is present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) - - -**Result:** warn - -**Remediation:** -Decide on an appropriate level for this parameter and set it, -either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. - -**Audit:** - -```bash -/bin/ps -fC kubelet -``` - -**Audit Config:** - -```bash -/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig -``` - -**Expected Result**: - -```console -'{.podPidsLimit}' is present -``` - -**Returned Value**: - -```console -apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key -``` - -## 5.1 RBAC and Service Accounts -### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) - - -**Result:** warn - -**Remediation:** -Identify all clusterrolebindings to the cluster-admin role. Check if they are used and -if they need this role or if they could use a role with fewer privileges. -Where possible, first bind users to a lower privileged role and then remove the -clusterrolebinding to the cluster-admin role : -kubectl delete clusterrolebinding [name] - -### 5.1.2 Minimize access to secrets (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove get, list and watch access to Secret objects in the cluster. - -### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) - - -**Result:** warn - -**Remediation:** -Where possible replace any use of wildcards in clusterroles and roles with specific -objects or actions. - -### 5.1.4 Minimize access to create pods (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to pod objects in the cluster. - -### 5.1.5 Ensure that default service accounts are not actively used. (Manual) - - -**Result:** warn - -**Remediation:** -Create explicit service accounts wherever a Kubernetes workload requires specific access -to the Kubernetes API server. -Modify the configuration of each default service account to include this value -automountServiceAccountToken: false - -### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) - - -**Result:** warn - -**Remediation:** -Modify the definition of pods and service accounts which do not need to mount service -account tokens to disable it. - -### 5.1.7 Avoid use of system:masters group (Manual) - - -**Result:** warn - -**Remediation:** -Remove the system:masters group from all users in the cluster. - -### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove the impersonate, bind and escalate rights from subjects. - -### 5.1.9 Minimize access to create persistent volumes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove create access to PersistentVolume objects in the cluster. - -### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the proxy sub-resource of node objects. - -### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. - -### 5.1.12 Minimize access to webhook configuration objects (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects - -### 5.1.13 Minimize access to the service account token creation (Manual) - - -**Result:** warn - -**Remediation:** -Where possible, remove access to the token sub-resource of serviceaccount objects. - -## 5.2 Pod Security Standards -### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that either Pod Security Admission or an external policy control system is in place -for every namespace which contains user workloads. - -### 5.2.2 Minimize the admission of privileged containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of privileged containers. - -### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostPID` containers. - -### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostIPC` containers. - -### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of `hostNetwork` containers. - -### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. - -### 5.2.7 Minimize the admission of root containers (Manual) - - -**Result:** warn - -**Remediation:** -Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` -or `MustRunAs` with the range of UIDs not including 0, is set. - -### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with the `NET_RAW` capability. - -### 5.2.9 Minimize the admission of containers with added capabilities (Automated) - - -**Result:** warn - -**Remediation:** -Ensure that `allowedCapabilities` is not present in policies for the cluster unless -it is set to an empty array. - -### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) - - -**Result:** warn - -**Remediation:** -Review the use of capabilites in applications running on your cluster. Where a namespace -contains applicaions which do not require any Linux capabities to operate consider adding -a PSP which forbids the admission of containers which do not drop all capabilities. - -### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. - -### 5.2.12 Minimize the admission of HostPath volumes (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers with `hostPath` volumes. - -### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) - - -**Result:** warn - -**Remediation:** -Add policies to each namespace in the cluster which has user workloads to restrict the -admission of containers which use `hostPort` sections. - -## 5.3 Network Policies and CNI -### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) - - -**Result:** warn - -**Remediation:** -If the CNI plugin in use does not support network policies, consideration should be given to -making use of a different plugin, or finding an alternate mechanism for restricting traffic -in the Kubernetes cluster. - -### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create NetworkPolicy objects as you need them. - -## 5.4 Secrets Management -### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) - - -**Result:** warn - -**Remediation:** -If possible, rewrite application code to read Secrets from mounted secret files, rather than -from environment variables. - -### 5.4.2 Consider external secret storage (Manual) - - -**Result:** warn - -**Remediation:** -Refer to the Secrets management options offered by your cloud provider or a third-party -secrets management solution. - -## 5.5 Extensible Admission Control -### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and setup image provenance. - -## 5.7 General Policies -### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) - - -**Result:** warn - -**Remediation:** -Follow the documentation and create namespaces for objects in your deployment as you need -them. - -### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) - - -**Result:** warn - -**Remediation:** -Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. -An example is as below: - securityContext: - seccompProfile: - type: RuntimeDefault - -### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) - - -**Result:** warn - -**Remediation:** -Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a -suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker -Containers. - -### 5.7.4 The default namespace should not be used (Manual) - - -**Result:** warn - -**Remediation:** -Ensure that namespaces are created to allow for appropriate segregation of Kubernetes -resources and that all new resources are created in a specific namespace. - diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security-best-practices.md b/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security-best-practices.md index fa958639c1f..beeb2888880 100644 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security-best-practices.md +++ b/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security-best-practices.md @@ -25,6 +25,6 @@ If you require such features, combine Layer 7 firewalls with [external authentic You should protect the following ports behind an [external load balancer](../../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-4-load-balancer) that has SSL offload enabled: - **K3s:** Port 6443, used by the Kubernetes API. -- **RKE and RKE2:** Port 6443, used by the Kubernetes API, and port 9345, used for node registration. +- **RKE2:** Port 6443, used by the Kubernetes API, and port 9345, used for node registration. These ports have TLS SAN certificates which list nodes' public IP addresses. An attacker could use that information to gain unauthorized access or monitor activity on the cluster. Protecting these ports helps mitigate against nodes' public IP addresses being disclosed to potential attackers. diff --git a/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security.md b/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security.md index f16699b8ac6..5d9ece208e7 100644 --- a/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security.md +++ b/versioned_docs/version-2.12/reference-guides/rancher-security/rancher-security.md @@ -31,22 +31,14 @@ On this page, we provide security related documentation along with resources to NeuVector is an open-source, container-focused security application that is now integrated into Rancher. NeuVector provides production security, DevOps vulnerability protection, and a container firewall, et al. Please see the [Rancher docs](../../integrations-in-rancher/neuvector/neuvector.md) and the [NeuVector docs](https://open-docs.neuvector.com/) for more information. -## Running a CIS Security Scan on a Kubernetes Cluster +## Running a Compliance Security Scan on a Kubernetes Cluster -Rancher leverages [kube-bench](https://github.com/aquasecurity/kube-bench) to run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the [CIS](https://www.cisecurity.org/cis-benchmarks/) (Center for Internet Security) Kubernetes Benchmark. +Rancher leverages [kube-bench](https://github.com/aquasecurity/kube-bench) to run a security scan to check whether Kubernetes is deployed according to security best practices. -The CIS Kubernetes Benchmark is a reference document that can be used to establish a secure configuration baseline for Kubernetes. - -The Center for Internet Security (CIS) is a 501(c\)(3) non-profit organization, formed in October 2000, with a mission to "identify, develop, validate, promote, and sustain best practice solutions for cyber defense and build and lead communities to enable an environment of trust in cyberspace". - -CIS Benchmarks are best practices for the secure configuration of a target system. CIS Benchmarks are developed through the generous volunteer efforts of subject matter experts, technology vendors, public and private community members, and the CIS Benchmark Development team. - -The Benchmark provides recommendations of two types: Automated and Manual. We run tests related to only Automated recommendations. - -When Rancher runs a CIS security scan on a cluster, it generates a report showing the results of each test, including a summary with the number of passed, skipped and failed tests. The report also includes remediation steps for any failed tests. - -For details, refer to the section on [security scans](../../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md). +When Rancher runs a Compliance scan on a cluster, it generates a report showing the results of each test, including a summary with the number of passed, skipped and failed tests. The report also includes remediation steps for any failed tests. +For details, refer to the section on [security scans](../../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md). +` ## SELinux RPM [Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. @@ -67,7 +59,7 @@ Each version of the hardening guide is intended to be used with specific version The benchmark self-assessment is a companion to the Rancher security hardening guide. While the hardening guide shows you how to harden the cluster, the benchmark guide is meant to help you evaluate the level of security of the hardened cluster. -Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. This guide will walk through the various controls and provide updated example commands to audit compliance in Rancher created clusters. The original benchmark documents can be downloaded from the [CIS website](https://www.cisecurity.org/benchmark/kubernetes/). +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. The original benchmark documents can be downloaded from the [CIS website](https://www.cisecurity.org/benchmark/kubernetes/). Each version of Rancher's self-assessment guide corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark. diff --git a/versioned_docs/version-2.12/reference-guides/rancher-webhook.md b/versioned_docs/version-2.12/reference-guides/rancher-webhook.md index 27c9b2b2e12..220be0f17eb 100644 --- a/versioned_docs/version-2.12/reference-guides/rancher-webhook.md +++ b/versioned_docs/version-2.12/reference-guides/rancher-webhook.md @@ -20,9 +20,7 @@ Each Rancher version is designed to be compatible with a single version of the w | Rancher Version | Webhook Version | Availability in Prime | Availability in Community | |-----------------|-----------------|-----------------------|---------------------------| -| v2.11.2 | v0.7.2 | ✓ | ✓ | -| v2.11.1 | v0.7.1 | ✓ | ✓ | -| v2.11.0 | v0.7.0 | ✗ | ✓ | +| v2.12.0 | v0.8.0 | ✗ | ✓ | ## Why Do We Need It? diff --git a/versioned_docs/version-2.12/reference-guides/single-node-rancher-in-docker/advanced-options.md b/versioned_docs/version-2.12/reference-guides/single-node-rancher-in-docker/advanced-options.md index c4dcde046d9..18cadd15b4c 100644 --- a/versioned_docs/version-2.12/reference-guides/single-node-rancher-in-docker/advanced-options.md +++ b/versioned_docs/version-2.12/reference-guides/single-node-rancher-in-docker/advanced-options.md @@ -44,6 +44,7 @@ Privileged access is [required.](../../getting-started/installation-and-upgrade/ docker run -d --restart=unless-stopped \ -p 80:80 -p 443:443 \ -v /var/log/rancher/auditlog:/var/log/auditlog \ + -e AUDIT_LOG_ENABLED=true \ -e AUDIT_LEVEL=1 \ --privileged \ rancher/rancher:latest diff --git a/versioned_docs/version-2.12/shared-files/_cluster-capabilities-table.md b/versioned_docs/version-2.12/shared-files/_cluster-capabilities-table.md index e53e3471aad..c4807fa5dba 100644 --- a/versioned_docs/version-2.12/shared-files/_cluster-capabilities-table.md +++ b/versioned_docs/version-2.12/shared-files/_cluster-capabilities-table.md @@ -8,7 +8,7 @@ | [Managing Projects, Namespaces and Workloads](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) | ✓ | ✓ | ✓ | ✓ | | [Using App Catalogs](../how-to-guides/new-user-guides/helm-charts-in-rancher/helm-charts-in-rancher.md) | ✓ | ✓ | ✓ | ✓ | | Configuring Tools ([Alerts, Notifiers, Monitoring](../integrations-in-rancher/monitoring-and-alerting/monitoring-and-alerting.md), [Logging](../integrations-in-rancher/logging/logging.md), [Istio](../integrations-in-rancher/istio/istio.md)) | ✓ | ✓ | ✓ | ✓ | -| [Running Security Scans](../how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | +| [Running Security Scans](../how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | | [Ability to rotate certificates](../how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md) | ✓ | ✓ | | | | Ability to [backup](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md) and [restore](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Rancher-launched clusters | ✓ | ✓ | | ✓4 | | [Cleaning Kubernetes components when clusters are no longer reachable from Rancher](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) | ✓ | | | | diff --git a/versioned_docs/version-2.12/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md b/versioned_docs/version-2.12/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md index adecdecde12..24fba32d46b 100644 --- a/versioned_docs/version-2.12/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md +++ b/versioned_docs/version-2.12/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md @@ -23,4 +23,4 @@ A Rancher Admin should now be able to view Rancher audit logs and follow through ## Feature Description - When Kubernetes Audit logs are enabled on the downstream cluster, in each event that is logged, the external Identity Provider's username is now logged for each request, at the "metadata" level. -- When Rancher API Audit logs are enabled on the Rancher installation, the external Identity Provider's username is also logged now at the `auditLog.level=1` for each request that hits the Rancher API server, including the login requests. +- When you enable Rancher API Audit logs for a Rancher installation, the external Identity Provider's username is also logged now at the `auditLog.level=0` for each request that hits the Rancher API server, including login requests. diff --git a/versioned_sidebars/version-2.12-sidebars.json b/versioned_sidebars/version-2.12-sidebars.json index 155f9b566e0..ab4ce77df65 100644 --- a/versioned_sidebars/version-2.12-sidebars.json +++ b/versioned_sidebars/version-2.12-sidebars.json @@ -215,7 +215,8 @@ "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml", - "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-generic-oidc" + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-generic-oidc", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-amazon-cognito" ] }, { @@ -300,7 +301,8 @@ "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates", "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry", - "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding" + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/notification-center" ] }, { @@ -500,6 +502,7 @@ "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster", "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster", "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-google-compute-engine-cluster", { "type": "category", "label": "Creating a VMware vSphere Cluster", @@ -527,7 +530,6 @@ } ] }, - "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences", "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents" ] }, @@ -725,21 +727,20 @@ }, { "type": "category", - "label": "CIS Scan Guides", + "label": "Compliance Scan Guides", "link": { "type": "doc", - "id": "how-to-guides/advanced-user-guides/cis-scan-guides/cis-scan-guides" + "id": "how-to-guides/advanced-user-guides/compliance-scan-guides/compliance-scan-guides" }, "items": [ - "how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark", - "how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark", - "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan", - "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule", - "how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests", - "how-to-guides/advanced-user-guides/cis-scan-guides/view-reports", - "how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark", - "how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", - "how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run" + "how-to-guides/advanced-user-guides/compliance-scan-guides/install-rancher-compliance", + "how-to-guides/advanced-user-guides/compliance-scan-guides/uninstall-rancher-compliance", + "how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan", + "how-to-guides/advanced-user-guides/compliance-scan-guides/run-a-scan-periodically-on-a-schedule", + "how-to-guides/advanced-user-guides/compliance-scan-guides/view-reports", + "how-to-guides/advanced-user-guides/compliance-scan-guides/enable-alerting-for-rancher-compliance", + "how-to-guides/advanced-user-guides/compliance-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", + "how-to-guides/advanced-user-guides/compliance-scan-guides/create-a-custom-compliance-version-to-run" ] }, { @@ -750,7 +751,6 @@ "id": "how-to-guides/advanced-user-guides/enable-experimental-features/enable-experimental-features" }, "items": [ - "how-to-guides/advanced-user-guides/enable-experimental-features/ui-server-side-pagination", "how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64", "how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers", "how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features", @@ -764,7 +764,9 @@ "how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters", "how-to-guides/advanced-user-guides/enable-user-retention", "how-to-guides/advanced-user-guides/enable-cluster-agent-scheduling-customization", - "how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer" + "how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer", + "how-to-guides/advanced-user-guides/configure-oidc-provider", + "how-to-guides/advanced-user-guides/ui-server-side-pagination" ] } ] @@ -904,7 +906,8 @@ "items": [ "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2", "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean", - "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure" + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure", + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/google-gce" ] } ] @@ -1003,49 +1006,6 @@ "id": "reference-guides/rancher-security/rancher-security" }, "items": [ - { - "type": "category", - "label": "Hardening Guides", - "link": { - "type": "doc", - "id": "reference-guides/rancher-security/hardening-guides/hardening-guides" - }, - "items": [ - { - "type": "category", - "label": "RKE Hardening Guides", - "link": { - "type": "doc", - "id": "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-hardening-guide" - }, - "items": [ - "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" - ] - }, - { - "type": "category", - "label": "RKE2 Hardening Guides", - "link": { - "type": "doc", - "id": "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-hardening-guide" - }, - "items": [ - "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" - ] - }, - { - "type": "category", - "label": "K3s Hardening Guides", - "link": { - "type": "doc", - "id": "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-hardening-guide" - }, - "items": [ - "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" - ] - } - ] - }, { "type": "category", "label": "SELinux RPM", @@ -1165,16 +1125,15 @@ }, { "type": "category", - "label": "CIS Scans", + "label": "Compliance Scans", "link": { "type": "doc", - "id": "integrations-in-rancher/cis-scans/cis-scans" + "id": "integrations-in-rancher/compliance-scans/compliance-scans" }, "items": [ - "integrations-in-rancher/cis-scans/configuration-reference", - "integrations-in-rancher/cis-scans/rbac-for-cis-scans", - "integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests", - "integrations-in-rancher/cis-scans/custom-benchmark" + "integrations-in-rancher/compliance-scans/configuration-reference", + "integrations-in-rancher/compliance-scans/rbac-for-compliance-scans", + "integrations-in-rancher/compliance-scans/custom-benchmark" ] }, { @@ -1254,7 +1213,6 @@ "faq/general-faq", "faq/deprecated-features", "faq/install-and-configure-kubectl", - "faq/dockershim", "faq/technical-items", "faq/security", "faq/container-network-interface-providers", @@ -1305,7 +1263,9 @@ "type": "category", "label": "Example Workflows", "items": [ - "api/workflows/projects" + "api/workflows/projects", + "api/workflows/kubeconfigs", + "api/workflows/tokens" ] }, "api/api-reference",